diff --git a/.gitignore b/.gitignore index f8f5dbcc27..95aef6d969 100644 --- a/.gitignore +++ b/.gitignore @@ -238,3 +238,6 @@ env.yaml responses_api_agents/cvdp_agent/deps/ resources_servers/cvdp/data/gym_agentic_code_generation_no_commercial.jsonl resources_servers/cvdp/data/gym_cvdp_nonagentic_code_generation_no_commercial.jsonl + +# Generated Terminal-Bench 4 dataset +/benchmarks/terminal_bench_4/data/ diff --git a/benchmarks/terminal_bench_4/README.md b/benchmarks/terminal_bench_4/README.md new file mode 100644 index 0000000000..973471a179 --- /dev/null +++ b/benchmarks/terminal_bench_4/README.md @@ -0,0 +1,95 @@ +# Terminal-Bench 4.0 + +[Official Terminal-Bench 4.0 benchmark](https://www.tbench.ai/news/terminal-bench-4-0) (66 tasks). + +## Profiles + +- `terminal_bench_4/miniswe`: mini-SWE **2.1.0** `DefaultAgent`, with a generic + text-action prompt, Gym Responses model adapter, and task-local MCP CLI. + +## Dataset and deployment + +The dataset is `terminal-bench/terminal-bench@4.0.0`, pinned to +`sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732`. +`manifest.json` retains all 52 CPU, 11 CPU Compose, and 3 H100 tasks and their +individual package digests. Preparation writes identities only; the resources +server validates the dataset and task digests before allocation. + +Set `OPENSANDBOX_DOMAIN` and `OPENSANDBOX_API_KEY` for one deployment. For the +established split deployment, set `OPENSANDBOX_DOMAIN_CPU`, +`OPENSANDBOX_API_KEY_CPU`, `OPENSANDBOX_DOMAIN_GPU`, and +`OPENSANDBOX_API_KEY_GPU`, then use `++tb4_split_sandbox_endpoints=true`. +Credentials resolve in server configuration; handoffs carry only a provider +alias, sandbox ID, and working directory. Keep resolved configs private. + +Agent and verifier select the endpoint independently from their official GPU +requirements. The selected GPU deployment must supply H100s; its unsupported +`gpu_type` filter is disabled explicitly. Each task retains its CPU, memory, +storage, GPU count, build budget, agent budget, and verifier deadline. + +Compose uses the existing digest-verified `compose-images.json`. It preserves +startup commands, users, dependency health checks, shared-memory requirements, +and sidecar artifact collection. Declared-port TCP forwarding for shared-network +mode is not full Linux namespace sharing. Required capabilities and privileged +setup must be supplied by the deployment. The explicit `nextjs-performance` +overlay `CIRCLE_NODE_TOTAL=3` matches its two-CPU allocation; disclose this runtime +adaptation in comparisons. No official task package or grader is edited. + +The OpenSandbox adapter maps separate-verifier `no-network` policies to deny-all +egress. The deployment must enforce that policy for hostname and direct-IP +traffic. Dynamic allowlists and offline Compose are not supported by this adapter. + +## Run + +```sh +gym eval prepare --benchmark terminal_bench_4/miniswe +gym eval run --benchmark terminal_bench_4/miniswe \ + --model-type vllm_model --model-url http://MODEL_HOST:8000/v1 \ + --model MODEL_NAME --output results/tb4/rollouts.jsonl --concurrency 8 \ + ++use_absolute_ip=true ++tb4_split_sandbox_endpoints=true +``` + +mini-SWE's loop runs in the agent worker and calls the Gym model server. +MCP tasks also need Python venv/pip +for its pinned task-local `mcp==1.29.0` client. + +Select tasks during preparation: + +```sh +gym eval prepare --benchmark terminal_bench_4/miniswe \ + '++prepare_script_args.task_names=[formal-crypto,interleaved-vigenere,ks-solver-cpp]' +``` + +Preparation also accepts `++prepare_script_args.category=cpu`, `compose`, or `gpu`. +Prepare again without filters for all tasks. The profile defaults to one attempt; +official leaderboard submissions use five. Scheduler allocations must cover +setup, the full official agent budget, and verification. + +## Infra validation + +For capped smoke runs, add `++tb4_max_steps=3` and +`++tb4_agent_max_timeout_sec=900`. The cap can only shorten the task's official +agent budget. Default runs have no step cap or timeout override. Installation +uses the separate 360-second harness-setup budget. Provider renewal keeps +resources alive without extending agent execution. + +Validate CPU, then Compose, then GPU. A grade of zero can be a healthy smoke +outcome; absent setup, model execution, grading, or required artifacts is not a +successful model run. +Infrastructure failures carry `infrastructure_error` and `_ng_failure_class` and +must be excluded from model-negative aggregates. + +The standalone smoke runner starts real Gym HTTP agent, resources, and model +servers on loopback. mini-SWE calls the Gym model server. +It requires the existing sandbox endpoint credentials and `OPENAI_API_KEY`. + +```sh +PYTHONPATH=. python benchmarks/terminal_bench_4/smoke.py \ + --harness miniswe --category cpu --env-file /path/to/private.env \ + --output results/tb4-smoke/cpu +``` + +Repeat for the Compose and GPU categories. `health.json` +requires both model-output evidence and an official grade. Inspect trajectories, +verifier output, and resource cleanup before promoting coverage. Capped runs are +not benchmark scores; missing submissions can exit grading before deeper tests. diff --git a/benchmarks/terminal_bench_4/compose-images.json b/benchmarks/terminal_bench_4/compose-images.json new file mode 100644 index 0000000000..8f67663ba5 --- /dev/null +++ b/benchmarks/terminal_bench_4/compose-images.json @@ -0,0 +1,558 @@ +{ + "apache/kafka-native:4.3.1": { + "architecture": "amd64", + "config": { + "Cmd": [ + "/etc/kafka/docker/run" + ], + "ExposedPorts": { + "9092/tcp": {} + }, + "User": "appuser", + "WorkingDir": "/" + }, + "config_digest": "sha256:a5ebb9f673af0a9cf23547dbf40831e0349bb3683af327366fc4ddf609394dd1", + "image": "apache/kafka-native@sha256:e50f6ad232dc9e524134ef6babba8285ba8a793326cb27ca2efc20fa83e45a0b", + "os": "linux" + }, + "harborframework/terminal-bench:ctr-optimization-environment-f22bf10e7c9a34e7@sha256:718822ca93c5d49694d69a094a84a68a8b6e3f09415ff23fb2baf01c209bd3c2": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:5c7c98966857d02d40d5214736eb4331173f44fcd95520cc50db9c8081169863", + "image": "harborframework/terminal-bench@sha256:718822ca93c5d49694d69a094a84a68a8b6e3f09415ff23fb2baf01c209bd3c2", + "os": "linux" + }, + "harborframework/terminal-bench:ctr-optimization-sidecar-api-35cd5fe7841b4933@sha256:4dee63e20e07c00cdab5b1ba977b1f848a074f967bd8f97c9cda9027725e17d5": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python", + "server.py" + ], + "ExposedPorts": { + "5000/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:ea006bea4a5016ffd8ab112d910062994fdda466024f65eb2ac924f3111619d5", + "image": "harborframework/terminal-bench@sha256:4dee63e20e07c00cdab5b1ba977b1f848a074f967bd8f97c9cda9027725e17d5", + "os": "linux" + }, + "harborframework/terminal-bench:cumulative-layout-shift-environment-7c5f839ce1345e4b@sha256:d840b99ee43fffbd7a76eab8e28d996063de8571725d2f4dbb5869d5d018baa1": { + "architecture": "amd64", + "config": { + "Cmd": [ + "node" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:f51684ef2aa90aa970c4268fa093510a589d0c07ba3ffa04d783aaed454cac8a", + "image": "harborframework/terminal-bench@sha256:d840b99ee43fffbd7a76eab8e28d996063de8571725d2f4dbb5869d5d018baa1", + "os": "linux" + }, + "harborframework/terminal-bench:cumulative-layout-shift-sidecar-barber-shop-data-backend-57bd45dbb96a662e@sha256:1f40268536b534165e32aa58b910eaa422a863c39cc6de005a09e3536efd9dda": { + "architecture": "amd64", + "config": { + "Cmd": [ + "pnpm", + "dev", + "--port", + "3001" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "ExposedPorts": { + "3001/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:c45945a3c439b29209fb05fc851cae0ccc5be56824186a751f3b60f804e589b4", + "image": "harborframework/terminal-bench@sha256:1f40268536b534165e32aa58b910eaa422a863c39cc6de005a09e3536efd9dda", + "os": "linux" + }, + "harborframework/terminal-bench:freight-dispatch-shift-environment-b8a25b24cf23c8d7@sha256:f9ffe49f11bcc6e77e33fd1bacd0a60027547526d716575c71e52517ac96daf8": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:87a2d29099c311730bfdf59df4635918b9211571a121a5ec62eb1201959f2318", + "image": "harborframework/terminal-bench@sha256:f9ffe49f11bcc6e77e33fd1bacd0a60027547526d716575c71e52517ac96daf8", + "os": "linux" + }, + "harborframework/terminal-bench:freight-dispatch-shift-sidecar-event-feed-42859b8ef057c829@sha256:f5975c403f3d3edc43e926ac49a63ea8561f9731ea63366180424380ad952782": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/srv/app/server.py" + ], + "ExposedPorts": { + "8080/tcp": {} + } + }, + "config_digest": "sha256:50c0916984fc812e54480d548d310d8b459011afb2999c3c17c54767ba094a68", + "image": "harborframework/terminal-bench@sha256:f5975c403f3d3edc43e926ac49a63ea8561f9731ea63366180424380ad952782", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-environment-760b960d24b30b4c@sha256:8245f02d4346749433b53cf2d2ea281e624c2410aa697394a8d70a32ae2c1a99": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:69c0819a66aeb68d1849bcf1e8ecfabdffd6b5df5d56cafe8aa40d6b861e7788", + "image": "harborframework/terminal-bench@sha256:8245f02d4346749433b53cf2d2ea281e624c2410aa697394a8d70a32ae2c1a99", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-sidecar-asset-ledger-61c5ee7eb0a19cd7@sha256:4906b2be733507125a12559fcb7a1162f483f397dd835bd66d94f463f2ef0da8": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/server.py" + ], + "ExposedPorts": { + "8001/tcp": {} + } + }, + "config_digest": "sha256:1cad0908720bd884cfc68342b63fc22e7843b1a94edf97ff3ca60ff833048b3d", + "image": "harborframework/terminal-bench@sha256:4906b2be733507125a12559fcb7a1162f483f397dd835bd66d94f463f2ef0da8", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-sidecar-compliance-ledger-55a4e36df409fe04@sha256:e7191e63c1f7609f9b1fd5c2bcc83511537f2b107df325acbbf7201d0d1c138f": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/server.py" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:543e91216e0a9ac488a84826b1c505bf0713cd835052fa50ca6e8ccccd5a675e", + "image": "harborframework/terminal-bench@sha256:e7191e63c1f7609f9b1fd5c2bcc83511537f2b107df325acbbf7201d0d1c138f", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-sidecar-document-vault-61247c2a9becac9d@sha256:2f309687721bdd0e55dd08193832051ebf064f6b84053f3ae6f6a79365687e34": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/server.py" + ], + "ExposedPorts": { + "8002/tcp": {} + } + }, + "config_digest": "sha256:453813dfa4eae58180f4e87e7e00158243733450b779b915f32c46cccd1c75d0", + "image": "harborframework/terminal-bench@sha256:2f309687721bdd0e55dd08193832051ebf064f6b84053f3ae6f6a79365687e34", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-sidecar-returns-ledger-0a15c968bfd78180@sha256:94a3e3f4423ddde7d84405c463d1738d1a680d63846ef5f7aca3d3dac88c1888": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/server.py" + ], + "ExposedPorts": { + "8003/tcp": {} + } + }, + "config_digest": "sha256:8e72004ec3e563fa053aea531b6c5b75bb906984ab8c573de2bd47e8c1cbe7f0", + "image": "harborframework/terminal-bench@sha256:94a3e3f4423ddde7d84405c463d1738d1a680d63846ef5f7aca3d3dac88c1888", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-sidecar-warranty-inbox-1b35b5032cd7e558@sha256:2e5e263df61ecd255c04952270ef3f3bb3c0bd2a819b210836661bdf5cfc3d61": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/server.py" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:5ffdfb0fa83e18fda941d838e728c97bb9a95cad49a24e5c557628ead44acb71", + "image": "harborframework/terminal-bench@sha256:2e5e263df61ecd255c04952270ef3f3bb3c0bd2a819b210836661bdf5cfc3d61", + "os": "linux" + }, + "harborframework/terminal-bench:heat-pump-warranty-sidecar-warranty-portal-9b1856ffd65950a9@sha256:c754da3670b3455e05c7a6f52608a0906e9b102cad2957b6a84e853461079a08": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/server.py" + ], + "ExposedPorts": { + "8000/tcp": {} + } + }, + "config_digest": "sha256:4b425918376ace7624d05b3804b83a030b6aaf88628faec483d2098a25dfc397", + "image": "harborframework/terminal-bench@sha256:c754da3670b3455e05c7a6f52608a0906e9b102cad2957b6a84e853461079a08", + "os": "linux" + }, + "harborframework/terminal-bench:intrastat-meldung-environment-8ee45061a2f8d64c@sha256:6357248f210a333fb08d0fc28f23020b909364ef1a67db132418e651e71bb788": { + "architecture": "amd64", + "config": { + "Cmd": [ + "sleep", + "infinity" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:8a3e84a403797711f9c2dd58799b1e0b137e994d9115f4bc13e02421619a9bfa", + "image": "harborframework/terminal-bench@sha256:6357248f210a333fb08d0fc28f23020b909364ef1a67db132418e651e71bb788", + "os": "linux" + }, + "harborframework/terminal-bench:intrastat-meldung-sidecar-compliance-hub-5ac013ecb7a2351d@sha256:c019cea99f27450679b0390571e6d68640dff055534b0f5c5e798fa30661138e": { + "architecture": "amd64", + "config": { + "Cmd": [ + "sleep", + "infinity" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:dc7df2898bc8c0147eb3463cbb4266a99e1fbd353b7a223a566712f2c6f84310", + "image": "harborframework/terminal-bench@sha256:c019cea99f27450679b0390571e6d68640dff055534b0f5c5e798fa30661138e", + "os": "linux" + }, + "harborframework/terminal-bench:intrastat-meldung-sidecar-dms-5ac013ecb7a2351d@sha256:5722eaf160ff537576d713a09e4c3ebd9b200d6b5054e02ab607e434a7dfd082": { + "architecture": "amd64", + "config": { + "Cmd": [ + "sleep", + "infinity" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:0e9ed3613c001dcef9ea59124ab5ae03b8209a578eeea9ad5cf895538f6e76a1", + "image": "harborframework/terminal-bench@sha256:5722eaf160ff537576d713a09e4c3ebd9b200d6b5054e02ab607e434a7dfd082", + "os": "linux" + }, + "harborframework/terminal-bench:intrastat-meldung-sidecar-idev-5ac013ecb7a2351d@sha256:b1ec3caa5ca80a7b2a00ec28ec54a51dbc7680ea297578a5bf27ab4ae79949e1": { + "architecture": "amd64", + "config": { + "Cmd": [ + "sleep", + "infinity" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:676fa251afee60cd57f59ef567da2de738177b9b89fd097bcc5f1588b203abfd", + "image": "harborframework/terminal-bench@sha256:b1ec3caa5ca80a7b2a00ec28ec54a51dbc7680ea297578a5bf27ab4ae79949e1", + "os": "linux" + }, + "harborframework/terminal-bench:intrastat-meldung-sidecar-odoo-5ac013ecb7a2351d@sha256:5b35fea6469419b2f2fafd9a562cbe9bab0d38decfdcdf27a3c8f8fc4acf9778": { + "architecture": "amd64", + "config": { + "Cmd": [ + "sleep", + "infinity" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:d8220fbb24824d552cfdfe4fa1733f7c2d75a9904ff405cc2f7b391c10f34bdb", + "image": "harborframework/terminal-bench@sha256:5b35fea6469419b2f2fafd9a562cbe9bab0d38decfdcdf27a3c8f8fc4acf9778", + "os": "linux" + }, + "harborframework/terminal-bench:intrastat-meldung-sidecar-services-5ac013ecb7a2351d@sha256:e7af68dd2519d09eefc6e68d714e4115ef35f67ca8b674507bd3f93ac0d3f9c0": { + "architecture": "amd64", + "config": { + "Cmd": [ + "sleep", + "infinity" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:f2c91d8865e178225548fcd8fe41fe770b3ea50f6b83ce2174e555d4e03a6187", + "image": "harborframework/terminal-bench@sha256:e7af68dd2519d09eefc6e68d714e4115ef35f67ca8b674507bd3f93ac0d3f9c0", + "os": "linux" + }, + "harborframework/terminal-bench:kv-live-surgery-environment-6e8d8c54429dfc9f@sha256:dd272ea1f98a6529bfee74e4a52330a44e414017a0cad756add94e23cac1473a": { + "architecture": "amd64", + "config": { + "Entrypoint": [ + "/app/entrypoint.sh" + ], + "ExposedPorts": { + "9000/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:3f19ab151068c6872cb23c993aa9040e5bfafd2d8cc8676d743f87d8f59918ba", + "image": "harborframework/terminal-bench@sha256:dd272ea1f98a6529bfee74e4a52330a44e414017a0cad756add94e23cac1473a", + "os": "linux" + }, + "harborframework/terminal-bench:kv-live-surgery-sidecar-loadgen-e42e01bb667f9fd9@sha256:ecb13e07d75c8d041ab0f6357d2d84cc6112fc1ce1a182a7de71f2ab420f3fd5": { + "architecture": "amd64", + "config": { + "Cmd": [ + "/app/loadgen" + ], + "ExposedPorts": { + "8080/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:225682683e490d1852f151069146f073a0edd1d973119d09e21bbcc621d7bdef", + "image": "harborframework/terminal-bench@sha256:ecb13e07d75c8d041ab0f6357d2d84cc6112fc1ce1a182a7de71f2ab420f3fd5", + "os": "linux" + }, + "harborframework/terminal-bench:legacy-utility-triage-environment-5cc1128c429e8d05@sha256:f599a42278c8fd1b0185933752c29c33d4dfa5850cbabd7e712e70654d96a643": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3" + ], + "WorkingDir": "/workspace" + }, + "config_digest": "sha256:eadec3084b599c2ed244740daa875e6289b48364d03d1ea3ccfd89176ef4165c", + "image": "harborframework/terminal-bench@sha256:f599a42278c8fd1b0185933752c29c33d4dfa5850cbabd7e712e70654d96a643", + "os": "linux" + }, + "harborframework/terminal-bench:legacy-utility-triage-sidecar-legacy-app-44fa7b126bb99cb6@sha256:f381021e014f4f36f619068edf0f3a2ed54b05cfa84245301afbedc2d9dd3042": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/app.py" + ], + "ExposedPorts": { + "8000/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:0bdcccd639798d23b94f62fd221d56ae5876617043742a35541dde8b3553d7c0", + "image": "harborframework/terminal-bench@sha256:f381021e014f4f36f619068edf0f3a2ed54b05cfa84245301afbedc2d9dd3042", + "os": "linux" + }, + "harborframework/terminal-bench:legacy-utility-triage-sidecar-legacy-workstation-e8775dad9c6bd514@sha256:706a7124caa138c3e8fc116af4413a637087869cd1dd7bf7e86c2069afa682a4": { + "architecture": "amd64", + "config": { + "Cmd": [ + "/entrypoint.sh" + ], + "ExposedPorts": { + "5900/tcp": {} + } + }, + "config_digest": "sha256:f8e450b38e5f23ada8dd55250e0f9c3c7314e7691d94fa87c4e5b92940fc0540", + "image": "harborframework/terminal-bench@sha256:706a7124caa138c3e8fc116af4413a637087869cd1dd7bf7e86c2069afa682a4", + "os": "linux" + }, + "harborframework/terminal-bench:live-database-cutover-environment-5997c5b486ffe2b0@sha256:dbd2997e95d6bdde237f94cefb8cdc45ef5c7bbd44ecde02052a6dffcde14054": { + "architecture": "amd64", + "config": { + "Entrypoint": [ + "/app/entrypoint.sh" + ], + "ExposedPorts": { + "8080/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:23d0f29195913a7e2d7f28a2f9ae0ca1a29e144349be1f6df01b4c9512ca7b13", + "image": "harborframework/terminal-bench@sha256:dbd2997e95d6bdde237f94cefb8cdc45ef5c7bbd44ecde02052a6dffcde14054", + "os": "linux" + }, + "harborframework/terminal-bench:live-database-cutover-sidecar-customer-53744832a5bce5e9@sha256:aa5a2a5c2b9c1e334125f0e5953e1b082ac61fdb4c2b1291c9171045b5af3dcf": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/multiproc.py" + ], + "ExposedPorts": { + "9000/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:6e6de8f0654ad3495c071cff7464acffbce526df0910b2c9c99106a3b0b84cf9", + "image": "harborframework/terminal-bench@sha256:aa5a2a5c2b9c1e334125f0e5953e1b082ac61fdb4c2b1291c9171045b5af3dcf", + "os": "linux" + }, + "harborframework/terminal-bench:live-database-cutover-sidecar-mysql-db-15531a3d85d91c9d@sha256:79bf81670ba7d98dfb04552ccad598331c5729b92b8311dfb1b5edcd84d6c55b": { + "architecture": "amd64", + "config": { + "Cmd": [ + "mysqld" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "ExposedPorts": { + "3306/tcp": {}, + "33060/tcp": {} + }, + "WorkingDir": "/" + }, + "config_digest": "sha256:0fee310be83263f247bf468bce8d81f175126eb560ea0559e8f749ea9f865aef", + "image": "harborframework/terminal-bench@sha256:79bf81670ba7d98dfb04552ccad598331c5729b92b8311dfb1b5edcd84d6c55b", + "os": "linux" + }, + "harborframework/terminal-bench:live-database-cutover-sidecar-postgres-db-42772d973839630b@sha256:20d349253b3b0ead577aaf2208de0d7648692e3064e0efe88e5125e0b63be8c9": { + "architecture": "amd64", + "config": { + "Cmd": [ + "postgres" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "ExposedPorts": { + "5432/tcp": {} + } + }, + "config_digest": "sha256:9a603c597d477e127814db3d0d5719c61758a6383a67a37c959f7595854ef83e", + "image": "harborframework/terminal-bench@sha256:20d349253b3b0ead577aaf2208de0d7648692e3064e0efe88e5125e0b63be8c9", + "os": "linux" + }, + "harborframework/terminal-bench:medical-claims-processing-environment-98fb909bc55e54a7@sha256:dc51c3e3da8c9b5758b095543f94f4c5082ff401a8d63414a2898708e787d541": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:77302584ea1843494d96d531e2de1785b3e4d9954635366e49982b1de9455dc3", + "image": "harborframework/terminal-bench@sha256:dc51c3e3da8c9b5758b095543f94f4c5082ff401a8d63414a2898708e787d541", + "os": "linux" + }, + "harborframework/terminal-bench:medical-claims-processing-sidecar-playwright-mcp-8867302f7b779be5@sha256:9a5080ae6a8d839875f4a5536145a279a9321721dcfd0ac15286de8caffb3aff": { + "architecture": "amd64", + "config": { + "Entrypoint": [ + "/app/entrypoint.sh" + ], + "User": "pwuser", + "WorkingDir": "/app" + }, + "config_digest": "sha256:9a20d29cc239d28039f4c49a202abdebd162274e341692be6dd98d2efb3c2aff", + "image": "harborframework/terminal-bench@sha256:9a5080ae6a8d839875f4a5536145a279a9321721dcfd0ac15286de8caffb3aff", + "os": "linux" + }, + "harborframework/terminal-bench:medical-claims-processing-sidecar-workspace-37f3799ffc402618@sha256:d18cb02cc2d98bb1a9d92ed5804792bf52347cee129dbfe1a5bc3c853afe268e": { + "architecture": "amd64", + "config": { + "Entrypoint": [ + "/app/entrypoint.sh" + ], + "ExposedPorts": { + "18073/tcp": {} + }, + "WorkingDir": "/app/backend" + }, + "config_digest": "sha256:607ed7b61262c70384236e6c24a09ab2aabca50fdc463787109b62e507f3401a", + "image": "harborframework/terminal-bench@sha256:d18cb02cc2d98bb1a9d92ed5804792bf52347cee129dbfe1a5bc3c853afe268e", + "os": "linux" + }, + "harborframework/terminal-bench:nextjs-performance-environment-6069ddb935a93aee@sha256:c2b1ae7f93b579cc3378d13b4e613ac7e8c0a5f06e4a295142dc2a07459399f0": { + "architecture": "amd64", + "config": { + "Cmd": [ + "node" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:f22389c9efc5d5a24792caa7b10101416079c9f50f7d6a050007d843363c55f6", + "image": "harborframework/terminal-bench@sha256:c2b1ae7f93b579cc3378d13b4e613ac7e8c0a5f06e4a295142dc2a07459399f0", + "os": "linux" + }, + "harborframework/terminal-bench:nextjs-performance-sidecar-warehouse-api-ad5ec787ca3cb3eb@sha256:1b3bc750097ca5004dc7c5315835702616926e8901c03b2fe8b3017c48cabbf9": { + "architecture": "amd64", + "config": { + "Cmd": [ + "node", + "server.js" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "ExposedPorts": { + "4100/tcp": {}, + "4101/tcp": {} + }, + "WorkingDir": "/srv/warehouse-api" + }, + "config_digest": "sha256:b6cf5480838f9ca12bd532e1497011d9ecbeed6fffb37243261c65dc691f389a", + "image": "harborframework/terminal-bench@sha256:1b3bc750097ca5004dc7c5315835702616926e8901c03b2fe8b3017c48cabbf9", + "os": "linux" + }, + "harborframework/terminal-bench:payments-pipeline-fix-environment-18a0d069e5ba5a20@sha256:be62f14692d1aafaaffe40c46a01b034baa53a6e333931f9705c343054398f7e": { + "architecture": "amd64", + "config": { + "Entrypoint": [ + "/app/entrypoint.sh" + ], + "WorkingDir": "/app" + }, + "config_digest": "sha256:d1021e5ea0de2c97334c0313791406ecb17cbc17208730e587444f821170ee96", + "image": "harborframework/terminal-bench@sha256:be62f14692d1aafaaffe40c46a01b034baa53a6e333931f9705c343054398f7e", + "os": "linux" + }, + "harborframework/terminal-bench:payments-pipeline-fix-sidecar-customer-7526dda72acc4b73@sha256:29995da6ac028ab31f407430596a9fa5328e74ba20ee018f998af9e354b5c7ad": { + "architecture": "amd64", + "config": { + "Cmd": [ + "python3", + "/app/main.py" + ], + "ExposedPorts": { + "9000/tcp": {} + }, + "WorkingDir": "/app" + }, + "config_digest": "sha256:e5ff2a1e228d51bad7d649559560450dc5f531c922b2a096c87e4f33028d3409", + "image": "harborframework/terminal-bench@sha256:29995da6ac028ab31f407430596a9fa5328e74ba20ee018f998af9e354b5c7ad", + "os": "linux" + }, + "harborframework/terminal-bench:payments-pipeline-fix-sidecar-seeder-dfa2b554b10ec940@sha256:d74e2040609ab0c266aa8d36f7927f6966d393e5b28b0a594428b4b48e589259": { + "architecture": "amd64", + "config": { + "Entrypoint": [ + "python3", + "/seed/seed_history.py" + ], + "WorkingDir": "/seed" + }, + "config_digest": "sha256:e28c7cd610ce3e8504a93db910fe863630d459a947b068dff5f978332e3e4131", + "image": "harborframework/terminal-bench@sha256:d74e2040609ab0c266aa8d36f7927f6966d393e5b28b0a594428b4b48e589259", + "os": "linux" + }, + "redis:7-alpine": { + "architecture": "amd64", + "config": { + "Cmd": [ + "redis-server" + ], + "Entrypoint": [ + "docker-entrypoint.sh" + ], + "ExposedPorts": { + "6379/tcp": {} + }, + "WorkingDir": "/data" + }, + "config_digest": "sha256:5509c0097c6064aa8a3b1df58f1d950e67090fffa6678ae8f3f1dc2385f12deb", + "image": "library/redis@sha256:1db42ccef14898aa29bae778452d567534b59c107129cbc1163fb552de184d3c", + "os": "linux" + } +} diff --git a/benchmarks/terminal_bench_4/manifest.json b/benchmarks/terminal_bench_4/manifest.json new file mode 100644 index 0000000000..3e5343ce49 --- /dev/null +++ b/benchmarks/terminal_bench_4/manifest.json @@ -0,0 +1,337 @@ +{ + "dataset": "terminal-bench/terminal-bench", + "ref": "sha256:39d9f44b40420cde8fdcc087579c0d72a7e14fa3656d603c3f0d22fb35e27732", + "source_commit": "452bf305c6daa62fc59061d22133a7cbc7c1572e", + "tasks": [ + { + "name": "atrx-vep-crispr", + "ref": "sha256:4ebe9e13de8cbd39ee2f9e14a9d7b1233c047ae43fba238ebc6274465697be3a", + "category": "cpu" + }, + { + "name": "batched-eval-parity", + "ref": "sha256:e1d54abdb0a83b6c28714fe54c5fd0cfe053132cf2a9d511a0166c6c556ea2be", + "category": "cpu" + }, + { + "name": "biped-contact-dynamics", + "ref": "sha256:7d9120179572cbb6511108fcfa1e76e6bc1bb56f2cace8e606d4ccded15cb4c4", + "category": "cpu" + }, + { + "name": "bun-sourcemap-leak", + "ref": "sha256:e55ae542f338f50f4468d940c7d7c93a45d34fd418dc6cc5883c1fedc753603a", + "category": "cpu" + }, + { + "name": "cad-model", + "ref": "sha256:037b83651678230c04217bc7bd077c9e2c3d5a383bb368cdeb19e88a2202f644", + "category": "cpu" + }, + { + "name": "cargo-flight-dispatch", + "ref": "sha256:18a1c2670ce4be0e6752f48ed6c750281d7f97db025f5e74edc6c8728f62f5b5", + "category": "cpu" + }, + { + "name": "coq-block-bound", + "ref": "sha256:4e2a6b608b18eeda1e18996efa115fec3573d90d4f798406b073b078fea52c89", + "category": "cpu" + }, + { + "name": "ctr-optimization", + "ref": "sha256:df8b7d99e23ad23ac9c92da35ff0a2714d78781e9424edd63143cc86860e096b", + "category": "compose" + }, + { + "name": "cumulative-layout-shift", + "ref": "sha256:d9805202ade29c603a55f5aa5af27a794a2ed3897e05e8ff0a484b4a84a0ed8a", + "category": "compose" + }, + { + "name": "data-anonymization", + "ref": "sha256:dec4261252665cf2981783df104ea10c901dcab3720b27b4bf8909a059713f08", + "category": "cpu" + }, + { + "name": "distributed-dedup", + "ref": "sha256:a645441376fea3d5b1acc15b64fe288eb0bb835724f6bf7a21adb1eca2bb814d", + "category": "cpu" + }, + { + "name": "embedding-drift-monitor", + "ref": "sha256:6907ffce9ad5c8a8b41fe9c0a076a9aeb9754a2f8083c52f9aede5481a4f4c59", + "category": "cpu" + }, + { + "name": "fin-saccr-rwa", + "ref": "sha256:bd0ee47169986cfdd4b0ccc98f461382c612e1301dd75f98d1381015eb25398a", + "category": "cpu" + }, + { + "name": "foodstuff-beta-activity", + "ref": "sha256:52adca169e97e15fe8692986470b6556f478e7348a37c58739a5bdb3f02cc45a", + "category": "cpu" + }, + { + "name": "formal-crypto", + "ref": "sha256:cb4f5a280f6ac1bfcc6c919841810084e286ce3109969841e1f5739fe9f7acf7", + "category": "cpu" + }, + { + "name": "fp8-rmsnorm-gemm", + "ref": "sha256:c071d986784369f0403ebf60070c7f96d7df3acc7e5a4614c986f8f421acda38", + "category": "gpu" + }, + { + "name": "freecad-impeller", + "ref": "sha256:40d322e979d30dd53c0a9330445b8799cf5c02099b0718a3b3ad15231b366dba", + "category": "cpu" + }, + { + "name": "freecad-platform-drawing", + "ref": "sha256:ed6db89260074bd0db58b064d90977cc4d470843deefecd8a9caf7f21da9a747", + "category": "cpu" + }, + { + "name": "freecad-spring-clip", + "ref": "sha256:c3110552902a44e17326df0a87083eb6028068cfbc7aa47461c7f2c0aa528bab", + "category": "cpu" + }, + { + "name": "freight-dispatch-shift", + "ref": "sha256:9c27972688139fce999058386bd381ca194c9f4ba76161a52bb8560aee9ceb2c", + "category": "compose" + }, + { + "name": "glycan-ms2-elucidation", + "ref": "sha256:80bf0ef924d6ba9a4323172f5a13df1734609914ddd086deef2da0288e9c560d", + "category": "cpu" + }, + { + "name": "gsea-proteomics", + "ref": "sha256:7c0512e617cd4c0a0b595ebfb81258495d7e4a12c003cd0592c1137aa986a85d", + "category": "cpu" + }, + { + "name": "heat-pump-warranty", + "ref": "sha256:7ce04515367c33c3673d1e6dc387edfa9b5a2565e6f040b1d24626efc07d0ad7", + "category": "compose" + }, + { + "name": "hof-topology-interpenetration", + "ref": "sha256:2534e62c5ab3c8592f4e0de3dee14ff07a5c1c9434164e7f80e2f564273de014", + "category": "cpu" + }, + { + "name": "html-js-filter", + "ref": "sha256:f211960f1f7c66ace91c3d079db073ed0f841d458db4cad1f2f943cba6cc79f3", + "category": "cpu" + }, + { + "name": "interleaved-vigenere", + "ref": "sha256:238a75a32aad5e60b33a22ac53087790e95b752c6272f41d724407f198df4a14", + "category": "cpu" + }, + { + "name": "intrastat-meldung", + "ref": "sha256:d944c51392857e886211327f38086513c0540b5c2f557b75b2b44dec209b8222", + "category": "compose" + }, + { + "name": "jax-speedrun-gpu", + "ref": "sha256:b8ea491d376abbb0da944e7a3b0fdef710d6de76a85cd5e25d132a8a541748fe", + "category": "gpu" + }, + { + "name": "ks-solver-cpp", + "ref": "sha256:d4eef6bb296028e57be0abb4b19dab71cf04b807cbe22fad4bc64174f9380cc0", + "category": "cpu" + }, + { + "name": "kv-live-surgery", + "ref": "sha256:810958fbeefc6a7f677624f908561e6e47d88aec8c06fa5321d81f5aa75a9ffc", + "category": "compose" + }, + { + "name": "lake-temp-glm", + "ref": "sha256:4ccec50699de5de3b1bcdedf1189a56d72520220da27306c10d93419a3316a0c", + "category": "cpu" + }, + { + "name": "layout-config-recreation", + "ref": "sha256:997003ffc98f6df316167de8752c9beada20cb1292588b0fa6409d87ed3a1def", + "category": "cpu" + }, + { + "name": "layout-config-recreation2", + "ref": "sha256:193312b14b1dcb6ef551efff691cbb708b2032f58879b0b19efe88a2cddc9deb", + "category": "cpu" + }, + { + "name": "legacy-utility-triage", + "ref": "sha256:d9954d3b78c8440a5927bc9826bfd9a61827391bf47f0885575fc7f27c4c8b6e", + "category": "compose" + }, + { + "name": "live-database-cutover", + "ref": "sha256:850fc34a564f68c1cf6f48178700168a4e3322f7ac6de80df98eb8a42d54913f", + "category": "compose" + }, + { + "name": "math-eval-grader", + "ref": "sha256:aac7c54d58a56d79a5c0a0f869e70a74c42b327b086ba8010b0dd477e114b904", + "category": "gpu" + }, + { + "name": "medical-claims-processing", + "ref": "sha256:8c7adb0a16026a79de234ed834d03942b53bd70237d02f47a381dd295b880552", + "category": "compose" + }, + { + "name": "mp-checkpoint-consolidation", + "ref": "sha256:7be196e68aa533288fb3e6f78969816a9076ff4f1eaf6fca4c28be8eb619fc6c", + "category": "cpu" + }, + { + "name": "music-harmony", + "ref": "sha256:f32afbf135cbd3c622ee687ba032abe61cab85a33b5243ee5cbbde82557f3ed1", + "category": "cpu" + }, + { + "name": "mvcc-lsm-compaction", + "ref": "sha256:370406dd79cc164e50b73568de37c34eba1b0e30782d7dc03f87c38c68123137", + "category": "cpu" + }, + { + "name": "nextjs-performance", + "ref": "sha256:81f8315fb984f91208c45b4ac153c8801d07d3beec5eb0f0d3fa9d49b6588fdd", + "category": "compose" + }, + { + "name": "ontology-kg-querying", + "ref": "sha256:3149354c3fdc71ee698752c294cd9b020a908ab01280f7494513e69708f8833c", + "category": "cpu" + }, + { + "name": "payments-pipeline-fix", + "ref": "sha256:ed92bf0b59203522407a2f628f186ccd11ee300a7a1ee04641f9510e46db1f14", + "category": "compose" + }, + { + "name": "photonic-waveguide-routing", + "ref": "sha256:31cd14c817cd8b05f7fce5492bf8c44777b6bfc7c0e0057e482d18164d28423e", + "category": "cpu" + }, + { + "name": "pretrain-shard-corruption", + "ref": "sha256:0f17568e65b9e8b79480e474a457811a65815cdaf7272660efffd1bc690021a0", + "category": "cpu" + }, + { + "name": "production-planning", + "ref": "sha256:1958ac51553517b79e3a642ed4f3bd9affb5b42166dcbacaa4c32f4b040edaff", + "category": "cpu" + }, + { + "name": "protein-autointerp-disulfide", + "ref": "sha256:6b70d5b8b2ed87b10cd58461c58981d4296f504eda0869c63cd6b0fb2b9edfb9", + "category": "cpu" + }, + { + "name": "react-lead-form", + "ref": "sha256:790af6a6308be694896fa5258c3f3c1b536dfa75acdb9936799a4344a3e4573b", + "category": "cpu" + }, + { + "name": "retro-console-soc", + "ref": "sha256:2b43b109ac633a4740364a3942df1195a8ae6ebfdae93b6a53e1ade8a6ffcf9e", + "category": "cpu" + }, + { + "name": "risk-scorer-replay", + "ref": "sha256:3645f552b5485633a67027130c9d471eb1a364f1451b6ada140bacd148f9cd87", + "category": "cpu" + }, + { + "name": "roy-polymorph-cn", + "ref": "sha256:012e714e79b4ff4f0d36eeb8fa2b6fa3be1043a71af880af8a71f61487669dfc", + "category": "cpu" + }, + { + "name": "rs-archive-clone", + "ref": "sha256:a6e74dbc5290e535872d70d5a9f52b97ca040d520a7b5f11a6f5b14013c4da85", + "category": "cpu" + }, + { + "name": "satb-audio-transcription", + "ref": "sha256:2b1bf5dcc71458a7e378f84468959993a304a8e89e51381ac0f78163e6e60f14", + "category": "cpu" + }, + { + "name": "session-window-debug", + "ref": "sha256:f7e05e0d4c0e90d7050445fd158db514158272c50aafa1b2446ef3f68bd01182", + "category": "cpu" + }, + { + "name": "sglang-qwen-burst", + "ref": "sha256:32bdf75983144af64a855c3711df1136cd347219b05bf5ef3b839375b1f82941", + "category": "cpu" + }, + { + "name": "shadow-relay", + "ref": "sha256:bb4928fdbe32ef9a17fe829ae45270533a07c93e071bc81c3b250f5fc4aa49b6", + "category": "cpu" + }, + { + "name": "sound-change-cascade", + "ref": "sha256:6214ceb80aaa657d72658a6f39a44fa11ea51f443822b6a1212eef1ba65bf91d", + "category": "cpu" + }, + { + "name": "takens-embedding-lean", + "ref": "sha256:8d8c7bdc58a79af79502ac9cdaf4c65333ae907f08a9615f6b538fa0d7530ac4", + "category": "cpu" + }, + { + "name": "telecom-entity-resolution", + "ref": "sha256:f818fa0aabaa916c35966e67e611e06d2bed372555d4f16b0e147726d382d1ad", + "category": "cpu" + }, + { + "name": "uefi-bootkit", + "ref": "sha256:f17de5664e18ba15b6af2262c35954093fdfcfe7f2852e8ea57dd4a6f131a208", + "category": "cpu" + }, + { + "name": "vba-userform-port", + "ref": "sha256:38f141630a3803538908de5080d4a38e87fa3ce77a9fbbaad284cb7f9e89fe22", + "category": "cpu" + }, + { + "name": "vf2-speedup-networkx", + "ref": "sha256:cdbad1f303f57856199aae757bdbc6363111e8ad60d31db49c61d01706da63ab", + "category": "cpu" + }, + { + "name": "vllm-deepseek-streaming", + "ref": "sha256:cb78aec591411780aedf133b95ca829584ba48768bbc437487fa773afbde321d", + "category": "cpu" + }, + { + "name": "vpp-loss-divergence", + "ref": "sha256:6db99a33495fcd7489072a4753a365e0cb4db9ea64c2b3f85a8b44db984fd0d3", + "category": "cpu" + }, + { + "name": "wal-recovery-ordering", + "ref": "sha256:8eb700419711e0d5eba317b8f6dec41aa3411e03e320e82b8da8070974ff3aae", + "category": "cpu" + }, + { + "name": "wdm-design", + "ref": "sha256:99aa16fb8df2da75e69948e78ccefb69360c439d981011ce08efd57e7e918a91", + "category": "cpu" + } + ] +} diff --git a/benchmarks/terminal_bench_4/miniswe.yaml b/benchmarks/terminal_bench_4/miniswe.yaml new file mode 100644 index 0000000000..de5749055c --- /dev/null +++ b/benchmarks/terminal_bench_4/miniswe.yaml @@ -0,0 +1,25 @@ +config_paths: +- benchmarks/terminal_bench_4/resources.yaml +terminal_bench_4_miniswe: + responses_api_agents: + miniswe_sandboxed_agent: + entrypoint: app.py + domain: agent + description: mini-SWE 2.1 text-action profile on TB4 resources. + resources_server: + type: resources_servers + name: terminal_bench_4 + model_server: + type: responses_api_models + name: policy_model + step_limit: ${oc.select:tb4_max_steps,0} + datasets: + - name: terminal_bench_4 + type: benchmark + jsonl_fpath: benchmarks/terminal_bench_4/data/benchmark.jsonl + prepare_script: benchmarks/terminal_bench_4/prepare.py + num_repeats: 1 + sandbox_providers: + default: tb4_sandbox_default + cpu: tb4_sandbox_cpu + gpu: tb4_sandbox_gpu diff --git a/benchmarks/terminal_bench_4/prepare.py b/benchmarks/terminal_bench_4/prepare.py new file mode 100644 index 0000000000..dab9be5487 --- /dev/null +++ b/benchmarks/terminal_bench_4/prepare.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prepare pinned identities; the resources server resolves official packages.""" + +import json +from pathlib import Path + + +BENCHMARK_DIR = Path(__file__).parent +OUTPUT_PATH = BENCHMARK_DIR / "data" / "benchmark.jsonl" + + +def prepare(task_names: list[str] | None = None, category: str | None = None) -> Path: + manifest = json.loads((BENCHMARK_DIR / "manifest.json").read_text()) + tasks = manifest["tasks"] + if len(tasks) != 66 or len({task["name"] for task in tasks}) != 66: + raise ValueError("Terminal-Bench 4.0 must contain 66 unique tasks") + if task_names is not None: + unknown = set(task_names) - {task["name"] for task in tasks} + if unknown: + raise ValueError(f"Unknown TB4 task names: {sorted(unknown)}") + tasks = [task for task in tasks if task["name"] in task_names] + if category is not None: + if category not in {"cpu", "compose", "gpu"}: + raise ValueError(f"Unknown TB4 category: {category}") + tasks = [task for task in tasks if task["category"] == category] + if not tasks: + raise ValueError("The TB4 task selection is empty") + OUTPUT_PATH.parent.mkdir(parents=True, exist_ok=True) + with OUTPUT_PATH.open("w") as output: + for task in tasks: + output.write( + json.dumps( + { + "task_name": f"terminal-bench/{task['name']}", + "task_ref": task["ref"], + "dataset_ref": manifest["ref"], + "responses_create_params": {"input": []}, + } + ) + + "\n" + ) + return OUTPUT_PATH + + +if __name__ == "__main__": + prepare() diff --git a/benchmarks/terminal_bench_4/resources.yaml b/benchmarks/terminal_bench_4/resources.yaml new file mode 100644 index 0000000000..7f85a02bb9 --- /dev/null +++ b/benchmarks/terminal_bench_4/resources.yaml @@ -0,0 +1,118 @@ +terminal_bench_4: + resources_servers: + terminal_bench_4: + entrypoint: app.py + domain: coding + verified: false + description: Official pinned TB4 setup, state, separate verification, and cleanup. + environment: + sandbox_env: {} + image_rewrites: [] + workdir: null + efs_logs_init_image: python:3.13-slim + cpu_enforcement_policy: limit + memory_enforcement_policy: limit + exec_shell: bash -c + efs_logs_host_path: /mnt/efs/data/shared + sandbox_request_gpu_type: false + sandbox_split_endpoints: ${oc.select:tb4_split_sandbox_endpoints,false} + sandbox_env_by_task: + nextjs-performance: + CIRCLE_NODE_TOTAL: '3' + sandbox_provider_options: + resource_requests: limits + compose_image_configs: benchmarks/terminal_bench_4/compose-images.json + sandbox_provider: + opensandbox: + connection: + domain: ${oc.env:OPENSANDBOX_DOMAIN,localhost} + protocol: http + use_server_proxy: true + transport_backend: aiohttp + tls_verify: false + create: + retries: 5 + retry_delay_s: 5 + retry_max_delay_s: 60 + request_timeout_s: 120 + timeout_s: 150 + connect_attempt_timeout_s: 150 + renew_interval_s: 1800 + operations: + background_exec: true + retries: 5 + retry_delay_s: 5 + retry_max_delay_s: 60 + status_poll_timeout_s: 20 + networking: + enabled: true + loopback_forwarding: true + setup_command: "if ! command -v python3 >/dev/null 2>&1; then\n apt-get update && DEBIAN_FRONTEND=noninteractive\ + \ apt-get install -y --no-install-recommends python3;\nfi" + runtime_requirements: + shm_size_metadata_key: nemo.nvidia.com/shm + capability_metadata: + SYS_PTRACE: + nemo.nvidia.com/ptrace: 'true' + capability_probes: + SYS_PTRACE: awk '/^CapEff:/ { print $2 }' /proc/self/status | { read caps; test $((0x$caps & (1 + << 19))) -ne 0; } + sandbox_metadata: + nemo.nvidia.com/resources: custom + sandbox_ttl_s: 28800 + sandbox_ready_timeout_s: 1200 + default_exec_timeout_s: 64800 + artifacts_dir: ${oc.select:tb4_jobs_dir,results/terminal_bench_4/resources} + agent_max_timeout_sec: ${oc.select:tb4_agent_max_timeout_sec,null} + max_concurrent_sessions: ${oc.select:tb4_concurrency,8} +tb4_sandbox_default: + opensandbox: + connection: + domain: ${oc.env:OPENSANDBOX_DOMAIN,localhost} + protocol: http + use_server_proxy: true + transport_backend: aiohttp + tls_verify: false + api_key: ${oc.env:OPENSANDBOX_API_KEY,null} + create: + connect_attempt_timeout_s: 150 + operations: + background_exec: true + retries: 5 + retry_delay_s: 5 + retry_max_delay_s: 60 + status_poll_timeout_s: 20 +tb4_sandbox_cpu: + opensandbox: + connection: + domain: ${oc.env:OPENSANDBOX_DOMAIN_CPU,localhost} + protocol: http + use_server_proxy: true + transport_backend: aiohttp + tls_verify: false + api_key: ${oc.env:OPENSANDBOX_API_KEY_CPU,null} + create: + connect_attempt_timeout_s: 150 + operations: + background_exec: true + retries: 5 + retry_delay_s: 5 + retry_max_delay_s: 60 + status_poll_timeout_s: 20 +tb4_sandbox_gpu: + opensandbox: + connection: + domain: ${oc.env:OPENSANDBOX_DOMAIN_GPU,localhost} + protocol: http + use_server_proxy: true + transport_backend: aiohttp + tls_verify: false + api_key: ${oc.env:OPENSANDBOX_API_KEY_GPU,null} + create: + connect_attempt_timeout_s: 150 + operations: + background_exec: true + retries: 5 + retry_delay_s: 5 + retry_max_delay_s: 60 + status_poll_timeout_s: 20 diff --git a/benchmarks/terminal_bench_4/smoke.py b/benchmarks/terminal_bench_4/smoke.py new file mode 100644 index 0000000000..b91be41025 --- /dev/null +++ b/benchmarks/terminal_bench_4/smoke.py @@ -0,0 +1,245 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run real split-server smoke checks in CPU, Compose, GPU order. + +Uses loopback HTTP between Gym services. mini-SWE routes model calls through +the local Gym model server. +""" + +import argparse +import asyncio +import hashlib +import json +import os +import socket +import traceback +from datetime import datetime, timezone +from pathlib import Path + +import aiohttp +import uvicorn +from dotenv import dotenv_values +from omegaconf import OmegaConf + +from nemo_gym import global_config, server_utils +from nemo_gym.server_utils import BaseServerConfig, GlobalAIOHTTPAsyncClientConfig, ServerClient +from resources_servers.terminal_bench_4.app import TerminalBench4Config, TerminalBench4ResourcesServer +from responses_api_agents.miniswe_sandboxed_agent.app import MiniSWESandboxedAgent, MiniSWESandboxedConfig +from responses_api_models.openai_model.app import SimpleModelServer, SimpleModelServerConfig + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def load_baseline(paths): + """Later health files replace earlier observations for the same harness/task.""" + baseline = {} + for path in paths: + for row in json.loads(path.read_text()): + baseline[row["harness"], row["task"]] = bool(row["healthy"]) + return baseline + + +async def main(args): + baseline = load_baseline(args.baseline_health) + if args.env_file: + for key, value in dotenv_values(args.env_file).items(): + if value and (key.startswith("OPENSANDBOX_") or key == "OPENAI_API_KEY"): + os.environ[key] = value + root = Path(__file__).parent + args.output.mkdir(parents=True, exist_ok=True) + source_root = root.parents[1] + source_paths = [ + source_root / path + for path in [ + "nemo_gym/sandbox/api.py", + "nemo_gym/sandbox/agent.py", + "nemo_gym/sandbox/handoff.py", + "nemo_gym/sandbox/adapters/docker_compose.py", + "nemo_gym/sandbox/providers/opensandbox/provider.py", + "benchmarks/terminal_bench_4/resources.yaml", + "benchmarks/terminal_bench_4/manifest.json", + "benchmarks/terminal_bench_4/compose-images.json", + "resources_servers/terminal_bench_4/compose_config.py", + "resources_servers/terminal_bench_4/app.py", + "resources_servers/terminal_bench_4/lifecycle.py", + "resources_servers/terminal_bench_4/task.py", + "resources_servers/terminal_bench_4/environment.py", + "resources_servers/terminal_bench_4/shared_logs.py", + "resources_servers/terminal_bench_4/transfers.py", + "resources_servers/terminal_bench_4/collection.py", + "resources_servers/terminal_bench_4/verifier.py", + "responses_api_agents/miniswe_sandboxed_agent/app.py", + "responses_api_agents/miniswe_sandboxed_agent/mcp_client.py", + ] + ] + (args.output / "run.json").write_text( + json.dumps( + { + "started_at": datetime.now(timezone.utc).isoformat(), + "harness": args.harness, + "model": args.model, + "steps": args.steps, + "agent_timeout_sec": args.agent_timeout, + "category": args.category, + "tasks": args.tasks, + "excluded_tasks": args.exclude_tasks, + "baseline_health": [str(path) for path in args.baseline_health], + "source_sha256": { + str(path.relative_to(source_root)): hashlib.sha256(path.read_bytes()).hexdigest() + for path in source_paths + }, + }, + indent=2, + ) + ) + config = OmegaConf.load(root / "resources.yaml") + config.tb4_split_sandbox_endpoints = True + config.tb4_agent_max_timeout_sec = args.agent_timeout + config.tb4_jobs_dir = str(args.output / "resources") + config.tb4_max_steps = args.steps + agent_name = "terminal_bench_4_" + args.harness + profile = OmegaConf.load(root / f"{args.harness}.yaml") + del profile["config_paths"] + config = OmegaConf.merge(config, profile) + ports = {name: free_port() for name in ["terminal_bench_4", agent_name, "policy_model"]} + resource_config = config.terminal_bench_4.resources_servers.terminal_bench_4 + resource_config.task_download_dir = str(args.task_cache) + resource_config.environment.sandbox_metadata["nemo-gym.nvidia.com/run"] = args.output.name + agent_config = next(iter(config[agent_name].responses_api_agents.values())) + model_config = { + "host": "127.0.0.1", + "port": ports["policy_model"], + "name": "policy_model", + "entrypoint": "app.py", + "openai_base_url": args.model_url, + "openai_api_key": os.environ["OPENAI_API_KEY"], + "openai_model": args.model, + } + config.policy_model = {"responses_api_models": {"openai_model": model_config}} + for name, values in [("terminal_bench_4", resource_config), (agent_name, agent_config)]: + values.update({"name": name, "host": "127.0.0.1", "port": ports[name]}) + config = OmegaConf.create(OmegaConf.to_container(config, resolve=True)) + global_config._GLOBAL_CONFIG_DICT = config + client = ServerClient(head_server_config=BaseServerConfig(host="127.0.0.1", port=1), global_config_dict=config) + http = server_utils.set_global_aiohttp_client(GlobalAIOHTTPAsyncClientConfig()) + resource = TerminalBench4ResourcesServer( + config=TerminalBench4Config.model_validate(OmegaConf.to_container(resource_config, resolve=True)), + server_client=client, + ) + agent = MiniSWESandboxedAgent( + config=MiniSWESandboxedConfig.model_validate(OmegaConf.to_container(agent_config, resolve=True)), + server_client=client, + ) + model = SimpleModelServer(config=SimpleModelServerConfig.model_validate(model_config), server_client=client) + servers = [ + uvicorn.Server( + uvicorn.Config(instance.setup_webserver(), host="127.0.0.1", port=ports[name], log_level="warning") + ) + for name, instance in [("terminal_bench_4", resource), (agent_name, agent), ("policy_model", model)] + ] + workers = [asyncio.create_task(s.serve()) for s in servers] + while not all(s.started for s in servers): + if any(t.done() for t in workers): + raise RuntimeError("A smoke server failed to start") + await asyncio.sleep(0.1) + manifest = json.loads((root / "manifest.json").read_text()) + rows = [] + semaphore = asyncio.Semaphore(args.concurrency) + + async def check(task): + async with semaphore: + started = datetime.now(timezone.utc).isoformat() + print(f"START {args.harness} {task['category']} {task['name']}", flush=True) + row = { + "task": task["name"], + "category": task["category"], + "started_at": started, + "harness": args.harness, + "model": args.model, + } + try: + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None)) as session: + async with session.post( + f"http://127.0.0.1:{ports[agent_name]}/run", + json={ + "task_name": "terminal-bench/" + task["name"], + "task_ref": task["ref"], + "dataset_ref": manifest["ref"], + "rollout_id": f"{args.output.name}-{task['name']}", + "responses_create_params": {"input": [], "max_output_tokens": 16384}, + }, + ) as response: + text = await response.text() + (args.output / f"{task['name']}.json").write_text(text) + if response.status != 200: + raise RuntimeError(f"Agent HTTP {response.status}; see task response") + result = json.loads(text) + agent_observed = bool((result.get("response") or {}).get("output")) + row |= { + "reward": result.get("reward"), + "evaluation_completed": result.get("evaluation_completed"), + "termination": result.get("termination"), + "infrastructure_error": result.get("infrastructure_error"), + "agent_observed": agent_observed, + "healthy": bool( + result.get("evaluation_completed") + and not result.get("infrastructure_error") + and agent_observed + ), + } + except Exception as exc: + row |= {"healthy": False, "error": str(exc)} + (args.output / f"{task['name']}.error.txt").write_text(traceback.format_exc()) + row["baseline_healthy"] = baseline.get((args.harness, task["name"])) + row["regression"] = not row["healthy"] and row["baseline_healthy"] is not False + rows.append(row) + (args.output / "health.json").write_text(json.dumps(rows, indent=2)) + print(f"END {task['name']} healthy={row['healthy']} reward={row.get('reward')}", flush=True) + + try: + for category in ["cpu", "compose", "gpu"]: + tasks = [ + t + for t in manifest["tasks"] + if t["category"] == category + and (not args.tasks or t["name"] in args.tasks) + and t["name"] not in args.exclude_tasks + and (not args.category or args.category == category) + ] + await asyncio.gather(*(check(t) for t in tasks)) + if any(row["regression"] for row in rows if row["category"] == category): + print( + f"Category {category} has new or unbaselined unhealthy tasks; inspect health.json before the next stage.", + flush=True, + ) + break + finally: + for server in servers: + server.should_exit = True + await asyncio.gather(*workers, return_exceptions=True) + await http.close() + server_utils._GLOBAL_AIOHTTP_CLIENT = None + return bool(rows) and not any(row["regression"] for row in rows) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--harness", choices=["miniswe"], default="miniswe") + parser.add_argument("--tasks", nargs="*") + parser.add_argument("--exclude-tasks", nargs="*", default=[]) + parser.add_argument("--category", choices=["cpu", "compose", "gpu"]) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--env-file", type=Path) + parser.add_argument("--baseline-health", type=Path, nargs="*", default=[]) + parser.add_argument("--task-cache", type=Path, default=Path("/tmp/tb4-packages")) + parser.add_argument("--model", default="gpt-5.4-mini-2026-03-17") + parser.add_argument("--model-url", default="https://api.openai.com/v1") + parser.add_argument("--steps", type=int, default=3) + parser.add_argument("--agent-timeout", type=int, default=900) + parser.add_argument("--concurrency", type=int, default=3) + raise SystemExit(0 if asyncio.run(main(parser.parse_args())) else 1) diff --git a/nemo_gym/sandbox/adapters/docker_compose.py b/nemo_gym/sandbox/adapters/docker_compose.py index 51912fcc28..8476f996fa 100644 --- a/nemo_gym/sandbox/adapters/docker_compose.py +++ b/nemo_gym/sandbox/adapters/docker_compose.py @@ -609,15 +609,24 @@ async def _stop(self): await sandbox.stop() except Exception as error: errors.append(error) + if self._volume_helper: + try: + result = await self._volume_helper.exec(f"rm -rf /tmp/compose-volumes/{self.project}", user="root") + if result.return_code: + raise RuntimeError(f"Shared volume cleanup failed: {result.stderr}") + except Exception as error: + errors.append(error) + try: + await self._volume_helper.stop() + self._volume_helper = None + except Exception as error: + errors.append(error) + try: + await self.provider.aclose() + except Exception as error: + errors.append(error) if errors: raise ExceptionGroup("Compose cleanup failed", errors) - if self._volume_helper: - result = await self._volume_helper.exec(f"rm -rf /tmp/compose-volumes/{self.project}", user="root") - if result.return_code: - raise RuntimeError(f"Shared volume cleanup failed: {result.stderr}") - await self._volume_helper.stop() - self._volume_helper = None - await self.provider.aclose() self._closed = True async def __aenter__(self): diff --git a/nemo_gym/sandbox/agent.py b/nemo_gym/sandbox/agent.py new file mode 100644 index 0000000000..1f8c9a8233 --- /dev/null +++ b/nemo_gym/sandbox/agent.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Run an agent harness in a sandbox owned by a resources server. + +Use this lifecycle when task setup and grading need to control the same environment, +while a separate agent worker runs the harness and makes model calls. For example, +a benchmark can provision task-specific images and sidecars once, then hand the main +sandbox to different harnesses without duplicating provisioning or verifier logic. + +The resources server implements the seed/start/verify/cancel contract and owns the +execution deadline and sandbox teardown. The agent supplies setup and execution +callbacks; this module attaches a borrowed sandbox handle, propagates session +cookies, requests grading or cancellation, and releases the handle without deleting +the environment needed by the verifier. +""" + +import asyncio +from pathlib import Path +from time import time +from uuid import uuid4 + +from nemo_gym.openai_utils import NeMoGymEasyInputMessage, NeMoGymResponse +from nemo_gym.sandbox import AsyncSandbox, create_provider, resolve_provider_config +from nemo_gym.sandbox.handoff import AgentTermination, SandboxedSeedResponse +from nemo_gym.server_utils import SESSION_ID_KEY, get_global_config_dict, get_response_json, raise_for_status + + +def empty_response(body, model): + return NeMoGymResponse( + id="resp_" + uuid4().hex, + created_at=int(time()), + model=model, + object="response", + output=[], + tool_choice=body.tool_choice, + tools=body.tools, + parallel_tool_calls=body.parallel_tool_calls, + ) + + +async def run_borrowed(agent, request, body, *, setup, execute): + body = body.model_copy(deep=True) + cookies = dict(request.cookies) + + async def resources(path, payload): + nonlocal cookies + response = await agent.server_client.post( + server_name=agent.config.resources_server.name, + url_path=path, + json=payload, + cookies=cookies, + ) + await raise_for_status(response) + cookies |= response.cookies + return await get_response_json(response) + + payload = body.model_dump() + payload["rollout_id"] = ( + agent.rollout_id_from_run(body) or body.capture_rollout_id or payload.get("rollout_id") or uuid4().hex + ) + payload["client_session_id"] = request.session[SESSION_ID_KEY] + # A resources HTTP retry reuses this nonce; another /run invocation cannot + # start a second harness against the same active rollout. + payload["execution_id"] = uuid4().hex + seed = SandboxedSeedResponse.model_validate(await resources("/seed_session", payload)) + sandbox = None + started = False + response = empty_response(body.responses_create_params, agent.config.model_server.name) + termination = AgentTermination(reason="infrastructure_error", detail="Agent setup did not complete") + extra = {} + try: + providers = agent.config.sandbox_providers + if seed.sandbox.provider not in providers: + raise ValueError(f"Unconfigured sandbox provider alias: {seed.sandbox.provider}") + config = resolve_provider_config(providers[seed.sandbox.provider], get_global_config_dict()) + provider = create_provider(config) + try: + sandbox = await AsyncSandbox.connect( + seed.sandbox.model_dump(exclude={"provider"}), + provider=provider, + owns_sandbox=False, + ) + except BaseException: + await provider.aclose() + raise + await asyncio.wait_for(setup(sandbox, seed), timeout=seed.setup_timeout_sec) + budget = await resources("/start_session", {"session_id": seed.session_id}) + started = True + body.responses_create_params.input = [NeMoGymEasyInputMessage(role="user", content=seed.instruction)] + try: + response, termination, extra = await execute(sandbox, seed, budget["agent_timeout_sec"]) + except TimeoutError: + termination = AgentTermination(reason="timeout") + except asyncio.CancelledError: + termination = AgentTermination(reason="cancelled") + raise + except Exception as exc: + termination = AgentTermination(reason="infrastructure_error", detail=f"{type(exc).__name__}: {exc}") + finally: + directory = Path("results") / agent.config.name / seed.session_id + if directory.exists() and str(directory) not in termination.artifacts: + termination.artifacts.append(str(directory)) + try: + if started: + # Verification is shielded from a disconnected collector: resources + # persist the result and always retain ownership of teardown. + verify = asyncio.create_task( + resources( + "/verify", + { + **body.model_dump(), + "session_id": seed.session_id, + "response": response.model_dump(mode="json"), + "termination": termination.model_dump(), + }, + ) + ) + result = await asyncio.shield(verify) + else: + await asyncio.shield(resources("/cancel_session", {"session_id": seed.session_id})) + finally: + if sandbox is not None: + await sandbox.release() + return result | extra + + +def artifact_directory(agent_name, session_id): + path = Path("results") / agent_name / session_id + path.mkdir(parents=True, exist_ok=True) + return path diff --git a/nemo_gym/sandbox/api.py b/nemo_gym/sandbox/api.py index 30d2574f91..408fd15bc1 100644 --- a/nemo_gym/sandbox/api.py +++ b/nemo_gym/sandbox/api.py @@ -385,9 +385,11 @@ def __init__( spec: SandboxSpec | None = None, *, owns_provider: bool = True, + owns_sandbox: bool = True, ) -> None: self._provider = create_provider(provider) if isinstance(provider, Mapping) else provider self._owns_provider = owns_provider + self._owns_sandbox = owns_sandbox self._spec = spec self._handle: SandboxHandle | None = None self._stopped = True @@ -524,6 +526,9 @@ async def endpoint(self, port: int) -> SandboxEndpoint: return resolved async def stop(self) -> None: + if not self._owns_sandbox: + await self.release() + return if self._closed: return try: @@ -536,6 +541,21 @@ async def stop(self) -> None: self._closed = True self._closed = True + async def release(self) -> None: + """Close this client's transport without destroying the remote sandbox. + + Use for borrowed attachments. The resources owner retains its independent + provider, renewal tasks and responsibility for remote destruction. + """ + if self._closed: + return + try: + if self._owns_provider: + await self._provider.aclose() + finally: + self._stopped = True + self._closed = True + async def serialize(self, *, scope: str | None = None) -> dict[str, Any]: """Return a JSON descriptor another process can rebuild this box from. @@ -559,7 +579,12 @@ async def serialize(self, *, scope: str | None = None) -> dict[str, Any]: @classmethod async def connect( - cls, descriptor: Mapping[str, Any] | Any, *, provider: SandboxProvider, owns_provider: bool = True + cls, + descriptor: Mapping[str, Any] | Any, + *, + provider: SandboxProvider, + owns_provider: bool = True, + owns_sandbox: bool = True, ) -> "AsyncSandbox": """Rebuild a sandbox in this process from a descriptor produced by :meth:`serialize`, using ``provider`` (which must support connect).""" @@ -571,7 +596,12 @@ async def connect( handle = await provider.connect(descriptor) workdir = descriptor.get("workdir") if isinstance(descriptor, Mapping) else None ports = descriptor.get("ports", ()) if isinstance(descriptor, Mapping) else () - sandbox = cls(provider, SandboxSpec(workdir=workdir, ports=ports), owns_provider=owns_provider) + sandbox = cls( + provider, + SandboxSpec(workdir=workdir, ports=ports), + owns_provider=owns_provider, + owns_sandbox=owns_sandbox, + ) sandbox._handle = handle sandbox._stopped = False return sandbox diff --git a/nemo_gym/sandbox/handoff.py b/nemo_gym/sandbox/handoff.py new file mode 100644 index 0000000000..23a26b1e6d --- /dev/null +++ b/nemo_gym/sandbox/handoff.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Agent/resources handoff. Provider aliases are resolved privately by each worker.""" + +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field + +from nemo_gym.base_resources_server import BaseSeedSessionResponse, BaseVerifyRequest, BaseVerifyResponse + + +class SandboxConnection(BaseModel): + provider: str + # Provider credentials and connection settings never belong in this descriptor. + sandbox_id: str + workdir: str | None = None + + +class SandboxedSeedResponse(BaseSeedSessionResponse): + session_id: str + sandbox: SandboxConnection + instruction: str + user: str | int | None = None + agent_timeout_sec: float = Field(gt=0) + setup_timeout_sec: float = Field(gt=0) + mcp_servers: list[dict[str, Any]] = Field(default_factory=list) + skills_dir: str | None = None + + +class SessionRequest(BaseModel): + session_id: str + + +class AgentTermination(BaseModel): + reason: Literal["completed", "timeout", "nonzero_exit", "cancelled", "infrastructure_error"] + exit_code: int | None = None + detail: str | None = None + artifacts: list[str] = Field(default_factory=list) + + +class SandboxedVerifyRequest(BaseVerifyRequest, SessionRequest): + termination: AgentTermination + + +class SandboxedVerifyResponse(BaseVerifyResponse): + model_config = ConfigDict(extra="allow") + + session_id: str + evaluation_completed: bool + termination: AgentTermination + infrastructure_error: str | None = None + artifacts: dict[str, str] = Field(default_factory=dict) + timings: dict[str, Any] = Field(default_factory=dict) diff --git a/nemo_gym/sandbox/providers/opensandbox/provider.py b/nemo_gym/sandbox/providers/opensandbox/provider.py index 6c79c4a30c..6392f73dce 100644 --- a/nemo_gym/sandbox/providers/opensandbox/provider.py +++ b/nemo_gym/sandbox/providers/opensandbox/provider.py @@ -18,13 +18,14 @@ import hashlib import ipaddress import logging +import math import re import shlex from collections.abc import Mapping from dataclasses import dataclass, field, replace from datetime import timedelta from pathlib import Path, PurePosixPath -from typing import Any, Awaitable, Callable +from typing import Any, Awaitable, Callable, Literal from urllib.parse import urlsplit from nemo_gym.sandbox.attribution import RUN_KEY, log_attribution_once, resolve_attribution, resolve_run_id @@ -522,12 +523,19 @@ class OpenSandboxCreateConfig: skip_health_check: bool = False connect_attempt_timeout_s: float = 30.0 connect_poll_s: float = 2.0 + # Refresh a created sandbox's TTL while this provider owns its handle. + # This does not change task/command timeouts or renew borrowed handles. + renew_interval_s: float | None = None def __post_init__(self) -> None: if self.image_pull_policy is not None: validate_image_pull_policy(self.image_pull_policy) if self.timeout_s is not None and self.timeout_s <= 0: raise ValueError("create.timeout_s must be > 0") + if self.renew_interval_s is not None and ( + not math.isfinite(self.renew_interval_s) or self.renew_interval_s <= 0 + ): + raise ValueError("create.renew_interval_s must be finite and > 0") if self.retries < 0: raise ValueError("create.retries must be >= 0") if self.retry_delay_s < 0: @@ -619,9 +627,9 @@ class OpenSandboxProviderOptions: volumes: tuple[Mapping[str, Any], ...] = () skip_health_check: bool | None = None extensions: Mapping[str, str] = field(default_factory=dict) - # Scheduling requests (same keys as SandboxSpec.resources, which become the - # limits). Unset, the server applies the single resources map as both. - resource_requests: Mapping[str, Any] | None = None + # Scheduling requests use SandboxSpec.resources keys. "limits" explicitly + # mirrors each sandbox's limits; omission leaves defaulting to the server. + resource_requests: Mapping[str, Any] | Literal["limits"] | None = None @classmethod def from_mapping(cls, options: Mapping[str, Any] | None) -> "OpenSandboxProviderOptions": @@ -660,8 +668,12 @@ def from_mapping(cls, options: Mapping[str, Any] | None) -> "OpenSandboxProvider if not isinstance(extensions, Mapping): raise TypeError("OpenSandbox provider option 'extensions' must be a mapping") resource_requests = options.get("resource_requests") - if resource_requests is not None and not isinstance(resource_requests, Mapping): - raise TypeError("OpenSandbox provider option 'resource_requests' must be a mapping") + if ( + resource_requests is not None + and resource_requests != "limits" + and not isinstance(resource_requests, Mapping) + ): + raise TypeError("OpenSandbox provider option 'resource_requests' must be a mapping or 'limits'") return cls( image_auth=dict(image_auth) if image_auth is not None else None, @@ -671,7 +683,7 @@ def from_mapping(cls, options: Mapping[str, Any] | None) -> "OpenSandboxProvider volumes=tuple(dict(volume) for volume in volumes), skip_health_check=skip_health_check, extensions=_string_map(dict(extensions)), - resource_requests=dict(resource_requests) if resource_requests is not None else None, + resource_requests=dict(resource_requests) if isinstance(resource_requests, Mapping) else resource_requests, ) @@ -687,9 +699,10 @@ class OpenSandboxNetworkingConfig: @dataclass class OpenSandboxRuntimeRequirementsConfig: - """Operator-supplied capability probes and create-time shared-memory metadata key.""" + """Operator-supplied capability probes and create-time runtime metadata.""" capability_probes: dict[str, str] = field(default_factory=dict) + capability_metadata: dict[str, dict[str, str]] = field(default_factory=dict) shm_size_metadata_key: str | None = None @@ -733,20 +746,23 @@ def __init__( # Sessions own aiohttp clients that only close() releases: aclose() # sweeps any still open; ended ones are retired on the next create/attach. self._pty_sessions: set[Any] = set() + self._renewals: dict[str, asyncio.Task[None]] = {} def validate_runtime_requirements(self, *, cap_add: tuple[str, ...], shm_size: int | None) -> dict[str, str]: """Reject requirements without an operator-configured implementation.""" + metadata = {} for capability in cap_add: if not self._runtime_requirements.capability_probes.get(capability, "").strip(): raise NotImplementedError(f"OpenSandbox requires a capability probe for {capability!r}") + metadata.update(self._runtime_requirements.capability_metadata.get(capability, {})) if shm_size is not None: if isinstance(shm_size, bool) or not isinstance(shm_size, int) or shm_size <= 0: raise ValueError("shm_size must be a positive number of bytes") key = self._runtime_requirements.shm_size_metadata_key if not key: raise NotImplementedError("OpenSandbox shm_size requires runtime_requirements.shm_size_metadata_key") - return {key: str(shm_size)} - return {} + metadata[key] = str(shm_size) + return metadata async def configure_runtime( self, handle: SandboxHandle, *, cap_add: tuple[str, ...], shm_size: int | None @@ -1000,6 +1016,8 @@ async def _retire_closed_pty_sessions(self) -> None: async def aclose(self) -> None: """Close provider-owned resources.""" + for sandbox_id in list(self._renewals): + await self._stop_renewal(sandbox_id) # PTY sessions hold their own aiohttp clients, which the shared httpx # transport below does not cover. for session in list(self._pty_sessions): @@ -1076,6 +1094,11 @@ async def _await_sdk_operation( # callers pass their own predicate. is_retryable: Callable[[BaseException], bool] = _is_retryable_sdk_operation_error, ) -> Any: + renewal = self._renewals.get(sandbox_id) + if renewal is not None and renewal.done() and not renewal.cancelled(): + error = renewal.exception() + if error is not None: + raise RuntimeError(f"OpenSandbox lifetime renewal failed for sandbox {sandbox_id!r}") from error AsyncRetrying, retry_if_exception, stop_after_attempt, wait_random_exponential = _require_tenacity() retry_count = self._operations.retries if retries is None else retries max_attempts = retry_count + 1 @@ -1310,8 +1333,39 @@ async def endpoint( headers.update(resolved.headers) return SandboxEndpoint(endpoint=endpoint_url, headers=headers) + def _start_renewal(self, handle: SandboxHandle, ttl_s: int | float) -> None: + async def renew() -> None: + while True: + await asyncio.sleep(self._create.renew_interval_s) + await self._await_sdk_operation( + lambda: handle.raw.renew(timedelta(seconds=ttl_s)), + operation="renew", + sandbox_id=handle.sandbox_id, + timeout_s=self._connection.request_timeout_s or 60, + ) + + def report_failure(task: asyncio.Task[None]) -> None: + if not task.cancelled() and (error := task.exception()) is not None: + LOGGER.error("OpenSandbox lifetime renewal failed for sandbox %r: %r", handle.sandbox_id, error) + + task = asyncio.create_task(renew(), name=f"opensandbox-renew-{handle.sandbox_id}") + task.add_done_callback(report_failure) + self._renewals[handle.sandbox_id] = task + + async def _stop_renewal(self, sandbox_id: str) -> BaseException | None: + task = self._renewals.pop(sandbox_id, None) + if task is None: + return None + task.cancel() + result = (await asyncio.gather(task, return_exceptions=True))[0] + return result if isinstance(result, Exception) else None + async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: """Create a sandbox through ``opensandbox.Sandbox.create``.""" + if self._create.renew_interval_s is not None and ( + spec.ttl_s is None or self._create.renew_interval_s >= spec.ttl_s + ): + raise ValueError("create.renew_interval_s requires a longer, explicit sandbox ttl_s") Sandbox, _, _, _, _ = _require_opensandbox_sdk() options = OpenSandboxProviderOptions.from_mapping(spec.provider_options) @@ -1322,7 +1376,11 @@ async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: "extensions": self._resolve_extensions(options.extensions), "connection_config": self._connection_config(request_timeout_s=self._create.request_timeout_s), } - if options.resource_requests is not None: + if options.resource_requests == "limits": + # Match the SDK's defaults when a service has no explicit resources. + kwargs["resource"] = kwargs["resource"] or {"cpu": "1", "memory": "2Gi"} + kwargs["resource_requests"] = dict(kwargs["resource"]) + elif options.resource_requests is not None: kwargs["resource_requests"] = _resource_map(SandboxResources.from_mapping(options.resource_requests)) if spec.image is not None: kwargs["image"] = _to_image_spec(spec.image, options.image_auth) @@ -1384,6 +1442,8 @@ async def _create_once(self, spec: SandboxSpec) -> SandboxHandle: except Exception: await self._cleanup_failed_create_handle(created_handle) raise + if self._create.renew_interval_s is not None: + self._start_renewal(handle, spec.ttl_s) return handle async def _create_with_retries( @@ -1850,6 +1910,7 @@ async def download_file(self, handle: SandboxHandle, source_path: str, target_pa async def close(self, handle: SandboxHandle) -> None: """Terminate the sandbox and close local SDK resources.""" + renewal_error = await self._stop_renewal(handle.sandbox_id) async def kill_ignore_missing() -> None: # Terminate is idempotent: not-found means the sandbox is already @@ -1897,5 +1958,9 @@ async def kill_ignore_missing() -> None: f"close_error={close_error!r}" ) from stop_error raise stop_error + if renewal_error is not None: + raise RuntimeError( + f"OpenSandbox lifetime renewal failed for sandbox {handle.sandbox_id!r}" + ) from renewal_error if close_error is not None: return diff --git a/resources_servers/terminal_bench_4/README.md b/resources_servers/terminal_bench_4/README.md new file mode 100644 index 0000000000..bd06813aee --- /dev/null +++ b/resources_servers/terminal_bench_4/README.md @@ -0,0 +1,96 @@ +# TB4 resources server + +## Contract + +1. `/seed_session` accepts `task_name`, `task_ref`, `dataset_ref`, and `rollout_id`. + The task must match the configured manifest. Dataset rows cannot select local + paths or override grading instructions. The response includes a session ID, + main-sandbox descriptor, instruction, user, working directory, setup/agent + budgets, MCP declarations, and a task skills directory. +2. The agent attaches with `owns_sandbox=False`, installs/configures its harness, + and calls `/start_session`. Resources return the remaining official agent + budget. Repeating this call does not restart the clock. +3. `/verify` accepts the session ID, Gym response/usage, termination reason, and + artifact references. Resources stop tracked harness process groups, collect + task-declared state, run the official verifier, and destroy the + collection. Official zero and nonzero rewards survive agent failure/timeout. +4. `/cancel_session` ends an abandoned setup or running episode. A running + cancellation still permits official grading. Closing a borrowed client never + destroys the resources-owned sandbox. + +The states are preparing, ready, agent running, verifying, and closed. Session +cookies bind access to the originating resources session. The agent supplies its +stable `client_session_id` so retries before the initial cookie response also reuse +the same episode. Its `execution_id` stays fixed across resources HTTP retries; +another worker invocation for the same rollout is rejected before attachment. +Run one resources worker per artifact directory. Concurrent duplicate +seeds share one attempt; conflicting identities fail. Concurrent verification +requests share one lifecycle and the first accepted agent result. Completed +verification retries return the recorded response, including after restart. + +The server retains the Compose creator and its relay/volume ownership for the +whole lifecycle. Shutdown cancels preparation and drains or interrupts finalization, then awaits cleanup. Client +HTTP disconnection does not cancel the resources task. Abandoned setup expires +at the setup deadline; abandoned execution expires at the official agent deadline. +Abrupt process death stops renewal; provider TTL is the cleanup fallback. Active +persisted episodes are rejected after restart and cannot be resumed safely. Use a +new rollout identity for a new attempt, not a stale descriptor. + +`evaluation_completed` means an official reward was retrieved. A scored negative +has `reward=0` with no infrastructure error. A missing verifier result or an agent +infrastructure failure adds `infrastructure_error` and `_ng_failure_class`, even +when a reward was retrieved. Artifacts include the compatible trial directory and +worker trajectory references. Failure diagnostics are written before teardown. + +## Non-root Compose services + +When loading agent Compose YAML, two task-specific adaptations use the +[Compose extensions](../../fern/versions/latest/pages/infrastructure/sandbox/compose.mdx): + +- `medical-claims-processing`: `playwright-mcp` keeps `pwuser`, disables host-file + injection with `x-sandbox.hosts: []`, and resolves `BROWSER_URL` to the workspace + sandbox IP via `x-sandbox.resolve_environment`. +- `payments-pipeline-fix`: `kafka` keeps `appuser` and disables host-file injection. + Its single-broker controller uses localhost; clients retain the `kafka` alias + needed by the advertised listener. + +Both services use their image's default user, omitting the redundant explicit +`user` value copied from image metadata. This avoids the provider attempting +`su` from a non-root process to the same user. + +These changes apply only to the generated runtime YAML. Pinned task packages, +other services, and verifier environments retain their original configuration. + +## Shared EFS logs + +The benchmark profile sets `environment.efs_logs_host_path` to +`/mnt/efs/data/shared`. Each episode creates a unique EFS directory with separate +agent and verifier subdirectories mounted read-write at `/logs`. The image's +default UID/GID owns its log root with mode `755`; workloads keep their original +execution user. This allows non-root images to initialize their log directories +and keeps root verifier reward-directory protections effective. Compose mounts +these logs in `main`; sidecar mounts and collection order remain unchanged. + +A CPU helper (`environment.efs_logs_init_image`, configured as `python:3.13-slim`) +initializes ownership and remains alive until both workloads are stopped. It +reuses the collected `/logs/artifacts` archive through EFS after agent teardown, +avoiding its upload from the resources host to the verifier. The archive is +checked against its collected digest, data-filtered, and repacked just as in the +host transfer. Exclusions and the local artifact manifest/files remain intact. +Overlapping artifact declarations and unavailable snapshots use the existing +ordered host restore. Agent logs, undeclared files, and agent-written reward +files do not leak into the fresh verifier role. + +Both endpoint pools must expose the same EFS share. An endpoint that explicitly +rejects the host mount with `VOLUME::HOST_PATH_NOT_ALLOWED` uses the original +filesystem/transfer lifecycle, recording `efs_logs_fallback` in diagnostics. +This preserves existing healthy GPU tasks on deployments without EFS support; +it does not fix non-root log creation on those deployments. Other provisioning +errors remain errors. Set `efs_logs_host_path: null` to disable EFS explicitly. + +Normal completion, cancellation, and handled failures remove the owned EFS +directory after workload teardown and then destroy the helper. If workload +deletion fails, EFS data is retained to avoid deleting a live mount. Persistent +session records include the helper ID and exact EFS host path/subdirectory for +recovery. Provider TTL expires sandboxes after abrupt process death, but EFS data +requires separate cleanup in that case. diff --git a/resources_servers/terminal_bench_4/app.py b/resources_servers/terminal_bench_4/app.py new file mode 100644 index 0000000000..9c891ff7cf --- /dev/null +++ b/resources_servers/terminal_bench_4/app.py @@ -0,0 +1,275 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pinned TB4 HTTP handoff backed by resource-owned native operations.""" + +import asyncio +import hashlib +import json +import re +from contextlib import asynccontextmanager +from pathlib import Path +from typing import ClassVar, Literal +from uuid import uuid4 + +from fastapi import HTTPException, Request +from pydantic import BaseModel, Field + +from nemo_gym.base_resources_server import BaseResourcesServerConfig, ReverifyMode, SimpleResourcesServer +from nemo_gym.sandbox.handoff import ( + AgentTermination, + SandboxedSeedResponse, + SandboxedVerifyRequest, + SandboxedVerifyResponse, + SessionRequest, +) +from nemo_gym.server_utils import SESSION_ID_KEY, is_nemo_gym_fastapi_entrypoint +from resources_servers.terminal_bench_4 import lifecycle +from resources_servers.terminal_bench_4.environment import EnvironmentConfig +from resources_servers.terminal_bench_4.lifecycle import NATIVE_VERSION, Session +from resources_servers.terminal_bench_4.task import PackageLoader + + +BENCHMARK = Path(__file__).resolve().parents[2] / "benchmarks" / "terminal_bench_4" + + +class TerminalBench4Config(BaseResourcesServerConfig): + num_workers: Literal[1] = 1 + REVERIFY_MODE: ClassVar[ReverifyMode] = ReverifyMode.UNSUPPORTED + manifest_path: Path = BENCHMARK / "manifest.json" + artifacts_dir: Path = Path("results/terminal_bench_4/resources") + environment: EnvironmentConfig + agent_max_timeout_sec: float | None = Field(default=None, gt=0) + max_concurrent_sessions: int = Field(default=8, gt=0) + shutdown_timeout_sec: float = Field(default=30, ge=0) + task_download_dir: Path | None = None + + +class TerminalBench4SeedRequest(BaseModel): + task_name: str + task_ref: str + dataset_ref: str + rollout_id: str = Field(min_length=1, max_length=256) + client_session_id: str | None = Field(default=None, min_length=1, max_length=256) + execution_id: str | None = Field(default=None, min_length=1, max_length=256) + + +def atomic_json(path, value): + temporary = path.with_name(path.name + ".tmp") + temporary.write_text(json.dumps(value, indent=2, allow_nan=False)) + temporary.replace(path) + + +class TerminalBench4ResourcesServer(SimpleResourcesServer): + config: TerminalBench4Config + + def model_post_init(self, context): + super().model_post_init(context) + self._manifest = json.loads(self.config.manifest_path.read_text()) + self._tasks = {"terminal-bench/" + task["name"]: task for task in self._manifest["tasks"]} + self._sessions: dict[str, Session] = {} + self._by_identity: dict[str, str] = {} + self._slots = asyncio.Semaphore(self.config.max_concurrent_sessions) + self._loader = PackageLoader(self.config.task_download_dir) + self._closing = False + self.config.artifacts_dir.mkdir(parents=True, exist_ok=True) + + def setup_webserver(self): + app = super().setup_webserver() + app.post("/start_session")(self.start_session) + app.post("/cancel_session")(self.cancel_session) + parent_lifespan = app.router.lifespan_context + + @asynccontextmanager + async def lifespan(app): + try: + async with parent_lifespan(app) as state: + yield state + finally: + self._closing = True + await lifecycle.shutdown(list(self._sessions.values()), self.config.shutdown_timeout_sec) + + app.router.lifespan_context = lifespan + return app + + def _owner(self, request): + return hashlib.sha256( + request.session.get("tb4_client_session_id", request.session[SESSION_ID_KEY]).encode() + ).hexdigest() + + def _state_path(self, identity): + return self.config.artifacts_dir / f"{identity}.json" + + def _persist(self, session): + resources = [] + for env in (session.environment, session.verifier_environment, session.shared_logs): + if env is not None: + resources.append( + { + "session_id": env.session_id, + "closed": env.closed, + "resources": env.resources or env.resource_identities(), + "cleanup_errors": env.cleanup_errors, + } + ) + atomic_json( + self._state_path(session.identity), + { + "record_version": 1, + "runtime": "gym-tb4-native", + "runtime_version": NATIVE_VERSION, + "session_id": session.session_id, + "owner": session.owner, + "request": session.request.model_dump(), + "phase": session.phase, + "subphase": session.subphase, + "result": session.result, + "identity": session.identity, + "deadlines": session.deadlines, + "termination": session.termination.model_dump() if session.termination else None, + "verify_body": session.verify_body.model_dump(mode="json") if session.verify_body else None, + "verified_response": session.verified_response.model_dump(mode="json") + if session.verified_response + else None, + "resources": resources or session.recorded_resources, + "diagnostics": session.diagnostics, + }, + ) + lookup = self.config.artifacts_dir / f"{session.session_id}.state" + temporary = lookup.with_suffix(".tmp") + temporary.write_text(session.identity) + temporary.replace(lookup) + if session.directory.exists(): + atomic_json(session.directory / "result.json", session.result) + + def _new_session(self, identity, owner, body, session_id, **kwargs): + kwargs.setdefault("result", {"runtime": "gym-tb4-native", "runtime_version": NATIVE_VERSION}) + session = Session(identity, owner, body, session_id, self.config.artifacts_dir / session_id, **kwargs) + session.slots = self._slots + session.config = self.config + session.persist = lambda: self._persist(session) + return session + + async def seed_session(self, request: Request, body: TerminalBench4SeedRequest) -> SandboxedSeedResponse: + if self._closing: + raise HTTPException(503, "Resources server is shutting down") + task = self._tasks.get(body.task_name) + if task is None or body.task_ref != task["ref"] or body.dataset_ref != self._manifest["ref"]: + raise HTTPException(422, "Task identity does not match the configured dataset pin") + if body.client_session_id: + request.session["tb4_client_session_id"] = body.client_session_id + owner = self._owner(request) + identity = hashlib.sha256(f"{owner}:{body.rollout_id}".encode()).hexdigest() + session_id = self._by_identity.get(identity) + if session_id is None: + if self._state_path(identity).exists(): + raise HTTPException( + 409, "Recorded episode cannot be resumed; use its session ID to retry verification" + ) + session_id = "tb4-" + uuid4().hex + session = self._new_session(identity, owner, body, session_id) + self._by_identity[identity] = session_id + self._sessions[session_id] = session + session.persist() + session.preparation = asyncio.create_task(lifecycle.prepare_session(session, self._loader)) + session = self._sessions[session_id] + if session.request != body: + raise HTTPException(409, "Rollout identity is already bound to another task or worker execution") + await asyncio.shield(session.preparation) + if session.phase == "closed" or session.finalization is not None or session.seed is None: + raise HTTPException(409, {"message": "Episode closed", "result": session.result}) + return session.seed + + def _session(self, request, session_id): + session = self._sessions.get(session_id) + if session is None: + if not re.fullmatch(r"tb4-[a-f0-9]{32}", session_id): + raise HTTPException(404, "Unknown session") + lookup = self.config.artifacts_dir / f"{session_id}.state" + if lookup.exists(): + identity = lookup.read_text() + if not re.fullmatch(r"[a-f0-9]{64}", identity): + raise HTTPException(409, "Invalid recorded episode identity") + state = json.loads(self._state_path(identity).read_text()) + if state["owner"] != self._owner(request): + raise HTTPException(404, "Unknown session") + if state["phase"] != "closed": + raise HTTPException( + 409, "Resources process restarted; episode cannot resume; provider TTL applies" + ) + if state.get("record_version", 0) not in (0, 1): + raise HTTPException(409, "Unsupported recorded episode version") + # Both native and legacy closed records retain the small result + # subset used below. No Harbor result model is needed for replay. + session = self._new_session( + state["identity"], + state["owner"], + TerminalBench4SeedRequest.model_validate(state["request"]), + session_id, + phase="closed", + result=state.get("result") or {}, + deadlines=state.get("deadlines") or {}, + diagnostics=state.get("diagnostics") or [], + recorded_resources=state.get("resources") or [], + ) + if state.get("termination"): + session.termination = AgentTermination.model_validate(state["termination"]) + if state.get("verify_body"): + session.verify_body = SandboxedVerifyRequest.model_validate(state["verify_body"]) + if state.get("verified_response"): + session.verified_response = SandboxedVerifyResponse.model_validate(state["verified_response"]) + self._sessions[session_id] = session + if session is None or session.owner != self._owner(request): + raise HTTPException(404, "Unknown session") + return session + + async def start_session(self, request: Request, body: SessionRequest) -> dict: + return await lifecycle.start_session(self._session(request, body.session_id)) + + async def cancel_session(self, request: Request, body: SessionRequest) -> dict: + session = self._session(request, body.session_id) + finish = await lifecycle.request_finish(session, AgentTermination(reason="cancelled")) + if finish is not None: + await asyncio.shield(finish) + return {"session_id": body.session_id, "phase": "closed"} + + async def verify(self, request: Request, body: SandboxedVerifyRequest) -> SandboxedVerifyResponse: + session = self._session(request, body.session_id) + if session.verified_response is not None: + return session.verified_response + finish = await lifecycle.request_finish(session, body.termination, body) + if finish is not None: + await asyncio.shield(finish) + async with session.lock: + if session.verified_response is not None: + return session.verified_response + result = session.result or {} + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + completed = "reward" in rewards + termination = session.termination or body.termination + failure = None + if not completed: + failure = (result.get("exception_info") or {}).get("exception_type", "MissingOfficialReward") + elif termination.reason == "infrastructure_error": + failure = termination.detail or "Agent infrastructure failure" + session.verified_response = SandboxedVerifyResponse( + **session.verify_body.model_dump(exclude={"termination"}), + reward=float(rewards.get("reward", 0)), + evaluation_completed=completed, + termination=termination, + infrastructure_error=failure, + failure_reason=failure, + artifacts={"trial": str(session.directory)}, + timings={ + key: result.get(key) for key in ("environment_setup", "agent_setup", "agent_execution", "verifier") + }, + **({"_ng_failure_class": "infrastructure_error"} if failure else {}), + ) + session.persist() + return session.verified_response + + +if __name__ == "__main__": + TerminalBench4ResourcesServer.run_webserver() +elif is_nemo_gym_fastapi_entrypoint(__file__): + app = TerminalBench4ResourcesServer.run_webserver() diff --git a/resources_servers/terminal_bench_4/collection.py b/resources_servers/terminal_bench_4/collection.py new file mode 100644 index 0000000000..fc9e47cf7e --- /dev/null +++ b/resources_servers/terminal_bench_4/collection.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TB4 main hooks/artifacts, main stop, then sidecar hooks/artifacts.""" + +import json +import shlex +from functools import partial +from pathlib import Path + +from resources_servers.terminal_bench_4.transfers import download_dir, download_file + + +async def collect(environment, directory, diagnostics): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + task = environment.task.config + shared_logs = getattr(environment, "shared_logs", None) + collected_artifacts = task.collected_artifacts + entries, claims = [], [] + + async def hooks(main): + for hook in task.verifier.collect: + if (hook.service == "main") != main: + continue + record = {"operation": "collect_hook", "service": hook.service, "command": hook.command} + try: + result = await environment.exec( + hook.command, + service=hook.service, + timeout_sec=int(hook.timeout_sec), + user=hook.user, + ) + record.update(return_code=result.return_code, stdout=result.stdout, stderr=result.stderr) + except Exception as exc: + record["error"] = str(exc) + diagnostics.append(record) + + async def artifacts(main): + for artifact in collected_artifacts: + if (artifact.service in (None, "main")) != main: + continue + target = directory / artifact.host_path + record = { + "source": artifact.source, + "destination": "artifacts/" + artifact.host_path.as_posix(), + "service": artifact.service, + "exclude": list(artifact.exclude), + "type": "file" if Path(artifact.source).suffix else "directory", + "status": "skipped", + } + # Preserve the reference's same-source re-collection exception. + collision = any( + source != artifact.source + and (claimed == target or claimed in target.parents or target in claimed.parents) + for claimed, source in claims + ) + if not collision: + claims.append((target, artifact.source)) + try: + kind = await environment.exec( + f"test -d {shlex.quote(artifact.source)}", + service=artifact.service, + ) + if kind.return_code: + kind = await environment.exec( + f"test -d {shlex.quote(artifact.source)}", + service=artifact.service, + user="root", + ) + record["type"] = "directory" if kind.return_code == 0 else "file" + except Exception: + pass + try: + sandbox = environment.sandbox(artifact.service) + if record["type"] == "directory": + shared_archive = ( + shared_logs.collection_archive(artifact, collected_artifacts) if shared_logs else None + ) + digest = await download_dir( + sandbox, + artifact.source, + target, + exclude=artifact.exclude, + exec_command=partial(environment.exec, service=artifact.service), + shared_archive=shared_archive, + ) + if shared_archive: + shared_logs.retain_archive(digest, target) + else: + record["exclude"] = [] + await download_file(sandbox, artifact.source, target) + record["status"] = "ok" + except Exception as exc: + record["status"] = "failed" + diagnostics.append({"operation": "collect_artifact", "source": artifact.source, "error": str(exc)}) + entries.append(record) + (directory / "manifest.json").write_text(json.dumps(entries, indent=2)) + + await hooks(True) + await artifacts(True) + sidecars = {a.service for a in task.artifacts if a.service not in (None, "main")} + sidecars |= {hook.service for hook in task.verifier.collect if hook.service != "main"} + if sidecars: + try: + await environment.stop_main() + except Exception as exc: + diagnostics.append({"operation": "stop_main", "error": str(exc)}) + await hooks(False) + await artifacts(False) + return entries diff --git a/resources_servers/terminal_bench_4/compose_config.py b/resources_servers/terminal_bench_4/compose_config.py new file mode 100644 index 0000000000..f01e0afa91 --- /dev/null +++ b/resources_servers/terminal_bench_4/compose_config.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resolve published Harbor Compose inputs using recorded OCI startup metadata.""" + +import re +from copy import deepcopy +from typing import Any + + +def _compose_literals(value: Any) -> Any: + if isinstance(value, str): + escaped = value.replace("$$", "\x00") + if re.search(r"\$(?:\{|[A-Za-z_])", escaped): + raise ValueError("Resolve Compose environment substitutions before starting the sandbox collection") + return escaped.replace("\x00", "$") + if isinstance(value, list): + return [_compose_literals(item) for item in value] + if isinstance(value, dict): + return {key: _compose_literals(item) for key, item in value.items()} + return value + + +def _bytes(value: str | int) -> int: + if isinstance(value, int): + return value + match = re.fullmatch(r"(\d+(?:\.\d+)?)\s*([kmgt]?)(?:i?b)?", value.lower()) + if match is None: + raise ValueError(f"Invalid Compose memory size: {value!r}") + return int(float(match[1]) * 1024 ** " kmgt".index(match[2] or " ")) + + +def resolve_compose(document: dict, main_image: str, image_configs: dict) -> dict: + """Apply Harbor's prebuilt-image base and normalize the published task overlay. + + Image metadata is acquired and digest-checked upstream. No image registry or + Docker installation is needed while executing the benchmark. + """ + resolved = _compose_literals(deepcopy(document)) + if not isinstance(resolved, dict) or not isinstance(resolved.get("services"), dict): + raise ValueError("Compose requires a services mapping") + services = resolved["services"] + # This is Harbor's docker-compose-prebuilt.yaml base command, including for + # images with a service-starting ENTRYPOINT that subsequently execs it. + services["main"] = { + "image": main_image, + "command": ["sh", "-c", "sleep infinity"], + **services.get("main", {}), + } + for name, service in services.items(): + image = service.get("image") + if image not in image_configs: + raise ValueError(f"Service {name!r} has no recorded OCI startup metadata for {image!r}") + record = image_configs[image] + if (record["os"], record["architecture"]) != ("linux", "amd64"): + raise ValueError(f"Service {name!r} requires a supported Linux/amd64 image") + config = record["config"] + service["image"] = record["image"] + explicit_entrypoint = service.get("entrypoint") is not None + if not explicit_entrypoint: + service["entrypoint"] = config.get("Entrypoint") or [] + if service.get("command") is None: + service["command"] = [] if explicit_entrypoint else config.get("Cmd") or [] + if config.get("WorkingDir"): + service.setdefault("working_dir", config["WorkingDir"]) + if config.get("User"): + service.setdefault("user", config["User"]) + service["expose"] = list(dict.fromkeys([*service.get("expose", []), *config.get("ExposedPorts", {})])) + if isinstance(service.get("environment"), list): + environment = {} + for item in service["environment"]: + if "=" not in item: + raise ValueError(f"Service {name!r} has an unresolved environment variable: {item!r}") + key, value = item.split("=", 1) + environment[key] = value + service["environment"] = environment + if isinstance(service.get("depends_on"), list): + service["depends_on"] = { + dependency: {"condition": "service_started"} for dependency in service["depends_on"] + } + for key in ("shm_size", "mem_limit"): + if key in service: + service[key] = _bytes(service[key]) + if config.get("Healthcheck") and "healthcheck" not in service: + health = config["Healthcheck"] + service["healthcheck"] = {"test": health["Test"]} + for source, target in (("Interval", "interval"), ("Timeout", "timeout"), ("StartPeriod", "start_period")): + if health.get(source): + service["healthcheck"][target] = f"{health[source] / 1_000_000_000}s" + if health.get("Retries"): + service["healthcheck"]["retries"] = health["Retries"] + return resolved diff --git a/resources_servers/terminal_bench_4/configs/terminal_bench_4.yaml b/resources_servers/terminal_bench_4/configs/terminal_bench_4.yaml new file mode 100644 index 0000000000..cbf678d8f8 --- /dev/null +++ b/resources_servers/terminal_bench_4/configs/terminal_bench_4.yaml @@ -0,0 +1,9 @@ +config_paths: +- benchmarks/terminal_bench_4/resources.yaml +terminal_bench_4: + resources_servers: + terminal_bench_4: + entrypoint: app.py + domain: coding + verified: false + description: Official pinned TB4 setup, state, separate verification, and cleanup. diff --git a/resources_servers/terminal_bench_4/environment.py b/resources_servers/terminal_bench_4/environment.py new file mode 100644 index 0000000000..4b39457655 --- /dev/null +++ b/resources_servers/terminal_bench_4/environment.py @@ -0,0 +1,337 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Own the main sandbox or concrete Compose collection for one TB4 role.""" + +import asyncio +import json +import math +import os +import shlex +from copy import deepcopy +from dataclasses import replace +from pathlib import Path, PurePosixPath +from time import monotonic +from typing import Any, Literal + +import yaml +from pydantic import Field + +from nemo_gym.sandbox import ( + AsyncSandbox, + AsyncSandboxCompose, + SandboxResources, + SandboxSpec, + resolve_provider_config, + resolve_provider_metadata, + rewrite_image, +) +from resources_servers.terminal_bench_4.compose_config import resolve_compose +from resources_servers.terminal_bench_4.task import Settings, resolve_env + + +class EnvironmentConfig(Settings): + cpu_enforcement_policy: Literal["limit"] + memory_enforcement_policy: Literal["limit"] + sandbox_provider: dict[str, Any] + sandbox_metadata: dict[str, str] + sandbox_provider_options: dict[str, Any] + sandbox_env: dict[str, str] + sandbox_env_by_task: dict[str, dict[str, str]] + sandbox_request_gpu_type: bool + sandbox_split_endpoints: bool + compose_image_configs: Path | None + sandbox_ttl_s: float = Field(gt=0) + sandbox_ready_timeout_s: float = Field(gt=0) + default_exec_timeout_s: float = Field(gt=0) + exec_shell: str | None + image_rewrites: list[dict[str, str]] + workdir: str | None + efs_logs_host_path: str | None + efs_logs_init_image: str + + +class HealthcheckError(RuntimeError): + pass + + +class Environment: + def __init__(self, task, config: EnvironmentConfig, session_id, directory, *, verifier=False): + self.task = task + self.config = config + self.session_id = session_id + self.directory = Path(directory) + self.settings = task.config.verifier_environment if verifier else task.config.environment + self.environment_dir = task.path / ("tests" if verifier else "environment") + self.provider_config = deepcopy(config.sandbox_provider) + self.pool = "default" + self.main = None + self.compose = None + self.closed = False + self.cleanup_errors = [] + self.resources = [] + self._cleanup_task = None + self.shared_logs = None + self.efs_logs_fallback = None + self.log_role = "verifier" if verifier else "agent" + self.task_env = resolve_env(self.settings.env) + self.startup_env = ( + self.task_env | config.sandbox_env_by_task.get(task.name.split("/")[-1], {}) | config.sandbox_env + ) + self.uses_compose = (self.environment_dir / "docker-compose.yaml").is_file() + if not verifier and not self.uses_compose: + services = {a.service for a in task.config.artifacts} | {h.service for h in task.config.verifier.collect} + if services - {None, "main"}: + raise ValueError("Sidecar artifacts and hooks require Compose") + if config.sandbox_split_endpoints: + self.pool = "gpu" if self.settings.gpus else "cpu" + if "opensandbox" not in self.provider_config: + raise ValueError("Split endpoints require OpenSandbox") + connection = self.provider_config["opensandbox"].setdefault("connection", {}) + for key, suffix in (("domain", "DOMAIN"), ("api_key", "API_KEY")): + name = f"OPENSANDBOX_{suffix}_{self.pool.upper()}" + if not os.environ.get(name): + raise ValueError(f"Missing environment variable: {name}") + connection[key] = os.environ[name] + elif "opensandbox" in self.provider_config and os.environ.get("OPENSANDBOX_API_KEY"): + self.provider_config["opensandbox"].setdefault("connection", {}).setdefault( + "api_key", os.environ["OPENSANDBOX_API_KEY"] + ) + resolve_provider_config(self.provider_config) + if config.efs_logs_host_path and "opensandbox" not in self.provider_config: + raise ValueError("EFS logs require OpenSandbox") + if self.settings.network_mode == "no-network" and ( + self.uses_compose or "opensandbox" not in self.provider_config + ): + raise ValueError("Offline verification requires a single OpenSandbox environment") + if self.uses_compose and config.compose_image_configs is None: + raise ValueError("Compose requires verified image startup metadata") + + def build_spec(self): + settings, config = self.settings, self.config + metadata = { + "tb4-session": self.session_id, + "tb4-task": self.task.name.split("/")[-1], + **resolve_provider_metadata(self.provider_config), + **config.sandbox_metadata, + } + if self.pool != "default": + metadata["nemo-gym.nvidia.com/resource-pool"] = self.pool + options = deepcopy(config.sandbox_provider_options) + if self.shared_logs is not None: + volumes = list(options.get("volumes") or []) + for volume in volumes: + target, logs = PurePosixPath(volume.get("mountPath", "")), PurePosixPath("/logs") + if ( + target == logs + or target in logs.parents + or logs in target.parents + or volume.get("name") == "tb4-logs" + ): + raise ValueError("EFS logs conflict with a configured /logs mount") + volumes.append(self.shared_logs.volume(self.log_role)) + options["volumes"] = volumes + if settings.network_mode == "no-network": + options["network_policy"] = {"defaultAction": "deny", "egress": []} + return SandboxSpec( + image=rewrite_image(settings.docker_image, config.image_rewrites), + resources=SandboxResources( + cpu=settings.cpus, + memory_mib=settings.memory_mb, + disk_gib=math.ceil(settings.storage_mb / 1024) if settings.storage_mb else None, + gpu=settings.gpus or None, + gpu_type=settings.gpu_types[0] if settings.gpu_types and config.sandbox_request_gpu_type else None, + ), + ttl_s=config.sandbox_ttl_s, + ready_timeout_s=config.sandbox_ready_timeout_s, + workdir=config.workdir, + env=self.startup_env, + metadata=metadata, + provider_options=options, + ) + + async def start(self): + spec = self.build_spec() + if self.uses_compose: + image_path = self.config.compose_image_configs + if not image_path.is_absolute(): + image_path = Path(__file__).resolve().parents[2] / image_path + document = resolve_compose( + yaml.safe_load((self.environment_dir / "docker-compose.yaml").read_text()), + self.settings.docker_image, + json.loads(image_path.read_text()), + ) + if self.log_role == "agent": + if self.task.name == "terminal-bench/medical-claims-processing": + # pwuser cannot edit /etc/hosts; its only peer URL is the + # browser's initial workspace page (which uses relative URLs). + document["services"]["playwright-mcp"]["x-sandbox"] = { + "hosts": [], + "resolve_environment": ["BROWSER_URL"], + } + # Use the image's default pwuser without an explicit su. + document["services"]["playwright-mcp"].pop("user", None) + elif self.task.name == "terminal-bench/payments-pipeline-fix": + # This single broker uses localhost for its controller. + # Clients still resolve its advertised kafka:9092 address. + document["services"]["kafka"]["x-sandbox"] = {"hosts": []} + # Use the image's default appuser without an explicit su. + document["services"]["kafka"].pop("user", None) + if "opensandbox" in self.provider_config: + for service in document["services"].values(): + if service.get("shm_size") is not None: + service.setdefault("labels", {})["nemo.nvidia.com/shm"] = str(service["shm_size"]) + sidecars = {a.service for a in self.task.config.artifacts} | { + h.service for h in self.task.config.verifier.collect + } + if sidecars - {None, "main"} - document["services"].keys(): + raise ValueError("Artifact or collect hook references an unavailable Compose service") + path = self.directory / "sandbox" / f"{self.session_id}.yaml" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(yaml.safe_dump(document, sort_keys=False)) + self.compose = AsyncSandboxCompose( + resolve_provider_config(self.provider_config), + path, + service_specs={ + name: spec + if name == "main" + else replace( + spec, + resources=SandboxResources(), + env={}, + provider_options=deepcopy(self.config.sandbox_provider_options), + ) + for name in document["services"] + }, + timeout_s=self.config.sandbox_ready_timeout_s, + ) + await self.compose.start() + self.main = self.compose.services["main"] + else: + self.main = AsyncSandbox(resolve_provider_config(self.provider_config), spec) + try: + await self.main.start() + except Exception as exc: + # Older endpoints (including the current GPU deployment) reject + # host mounts before allocating a sandbox. Keep their existing + # lifecycle usable, but never mask other provisioning failures. + if ( + self.shared_logs is None + or "VOLUME::HOST_PATH_NOT_ALLOWED" not in str(exc) + or self.shared_logs.host_path not in str(exc) + ): + raise + await self.main.stop() + self.efs_logs_fallback = str(exc) + self.shared_logs = None + self.main = AsyncSandbox(resolve_provider_config(self.provider_config), self.build_spec()) + await self.main.start() + if self.shared_logs is not None: + await self.shared_logs.initialize_role(self) + result = await self.exec("mkdir -p /logs/agent /logs/verifier /logs/artifacts", timeout_sec=60) + if result.return_code: + raise RuntimeError(f"Failed to initialize task log directories: {result.stderr}") + # Published images with a build spec already contain these files. + if ( + not self.uses_compose + and not (self.environment_dir / "Dockerfile").exists() + and self.environment_dir.is_dir() + ): + from resources_servers.terminal_bench_4.transfers import upload_dir + + cwd = self.settings.workdir or (await self.exec("pwd")).stdout.strip() + await upload_dir(self.main, self.environment_dir, cwd) + + def sandbox(self, service=None): + if service not in (None, "main"): + if self.compose is None or service not in self.compose.services: + raise ValueError(f"Unavailable Compose service: {service}") + return self.compose.services[service] + if self.main is None: + raise RuntimeError("Main sandbox is not running") + return self.main + + async def exec(self, command, *, service=None, cwd=None, env=None, timeout_sec=None, user=None): + main = service in (None, "main") + shell = self.config.exec_shell if main else "sh -c" + if shell: + command = f"{shell} {shlex.quote(command)}" + persistent = self.task_env if main and not self.uses_compose else {} + return await self.sandbox(service).exec( + command, + cwd=cwd, + env=(persistent | (env or {})) or None, + timeout_s=timeout_sec if timeout_sec is not None else self.config.default_exec_timeout_s, + user=user, + ) + + async def main_connection(self): + descriptor = await self.main.serialize(scope="operate") + if "sandbox_id" not in descriptor: + raise ValueError("TB4 requires an ID-based connectable provider") + cwd = await self.exec("pwd", timeout_sec=30, user=self.task.config.agent.user) + if cwd.return_code: + raise RuntimeError("Unable to determine the task working directory") + return {"provider": self.pool, "sandbox_id": descriptor["sandbox_id"], "workdir": cwd.stdout.strip()} + + async def healthcheck(self): + hc = self.settings.healthcheck + if hc is None: + return + grace = monotonic() + hc.start_period_sec + failures = 0 + while True: + in_grace = monotonic() < grace + result = await self.exec(hc.command, timeout_sec=int(hc.timeout_sec)) + if result.return_code == 0: + return + if not in_grace: + failures += 1 + if failures >= hc.retries: + raise HealthcheckError(f"Healthcheck failed after {hc.retries} consecutive retries: {hc.command}") + await asyncio.sleep(hc.start_interval_sec if in_grace else hc.interval_sec) + + async def quiesce_agent(self, session_id): + pidfile = shlex.quote(f"/tmp/{session_id}.pids") + result = await self.exec( + f"if [ -f {pidfile} ]; then groups=$(cat {pidfile}); for p in $groups; do " + "case $p in ''|*[!0-9]*) exit 1;; esac; " + 'kill -TERM -- -"$p" 2>/dev/null || true; done; sleep 1; ' + 'for p in $groups; do if kill -0 -- -"$p" 2>/dev/null; then ' + 'kill -KILL -- -"$p" 2>/dev/null || exit 1; fi; done; sleep 1; fi', + timeout_sec=30, + user=self.task.config.agent.user, + ) + if result.return_code: + raise RuntimeError("Could not stop the external agent before artifact collection") + + async def stop_main(self): + await self.main.stop() + + def resource_identities(self): + members = self.compose.services if self.compose else {"main": self.main} + result = [] + for name, sandbox in members.items(): + handle = getattr(sandbox, "_handle", None) + if handle is not None: + result.append({"service": name, "provider": self.pool, "sandbox_id": handle.sandbox_id}) + if self.compose: + result.append({"compose_project": self.compose.project, "provider": self.pool}) + return result + + async def stop(self): + if self._cleanup_task is None: + self._cleanup_task = asyncio.create_task(self._stop()) + await asyncio.shield(self._cleanup_task) + + async def _stop(self): + self.resources = self.resource_identities() + try: + if self.compose is not None: + await self.compose.stop() + elif self.main is not None: + await self.main.stop() + self.closed = True + except Exception as exc: + self.cleanup_errors.append({"error": str(exc), "resources": self.resources}) + raise diff --git a/resources_servers/terminal_bench_4/lifecycle.py b/resources_servers/terminal_bench_4/lifecycle.py new file mode 100644 index 0000000000..d712aa2d7f --- /dev/null +++ b/resources_servers/terminal_bench_4/lifecycle.py @@ -0,0 +1,374 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resource-owned preparation, deadlines, one finalizer, and cleanup.""" + +import asyncio +from dataclasses import dataclass, field +from datetime import datetime, timezone +from pathlib import Path +from time import monotonic +from typing import Any, Callable + +from fastapi import HTTPException + +from nemo_gym.sandbox.handoff import AgentTermination, SandboxedSeedResponse +from resources_servers.terminal_bench_4.collection import collect +from resources_servers.terminal_bench_4.environment import Environment +from resources_servers.terminal_bench_4.shared_logs import SharedLogs +from resources_servers.terminal_bench_4.transfers import download_dir +from resources_servers.terminal_bench_4.verifier import restore, run_verifier + + +NATIVE_VERSION = "1" +SETUP_TIMEOUT_SEC = 360 + + +def now(): + return datetime.now(timezone.utc).isoformat() + + +@dataclass +class Session: + identity: str + owner: str + request: Any + session_id: str + directory: Path + phase: str = "preparing" + subphase: str | None = None + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + preparation: asyncio.Task | None = None + finalization: asyncio.Task | None = None + watchdog: asyncio.Task | None = None + task: Any = None + environment: Any = None + verifier_environment: Any = None + shared_logs: Any = None + seed: SandboxedSeedResponse | None = None + termination: AgentTermination | None = None + verify_body: Any = None + verified_response: Any = None + result: dict = field(default_factory=dict) + deadline: float | None = None + setup_deadline: float | None = None + deadlines: dict = field(default_factory=dict) + diagnostics: list = field(default_factory=list) + recorded_resources: list = field(default_factory=list) + owns_slot: bool = False + slots: Any = None + config: Any = None + persist: Callable = field(default=lambda: None, repr=False) + + +def exception(session, error, error_type=None): + record = {"exception_type": error_type or type(error).__name__, "exception_message": str(error)} + session.result.setdefault("exception_info", record) + session.diagnostics.append({"phase": session.phase, "subphase": session.subphase, **record}) + + +def stop_watchdog(session): + watchdog, session.watchdog = session.watchdog, None + if watchdog and watchdog is not asyncio.current_task(): + watchdog.cancel() + + +async def cleanup(session): + session.subphase = "cleanup" + session.persist() + for env in (session.environment, session.verifier_environment): + if env is None or env.closed: + continue + try: + await env.stop() + except Exception as exc: + exception(session, exc) + if session.shared_logs is not None: + try: + # A failed sandbox deletion must not race removal of its live mount. + await session.shared_logs.stop( + remove_data=all( + env is None or env.closed for env in (session.environment, session.verifier_environment) + ) + ) + except Exception as exc: + exception(session, exc) + if session.owns_slot: + session.slots.release() + session.owns_slot = False + for key in ("environment_setup", "agent_setup", "agent_execution", "verifier"): + if key in session.result: + session.result[key].setdefault("finished_at", now()) + session.result["diagnostics"] = session.diagnostics + session.result["finished_at"] = now() + session.phase = "closed" + session.subphase = None + stop_watchdog(session) + session.persist() + + +async def prepare_session(session, loader): + # The reference creates these even for providers without host mounts. In + # particular, the convention directory must remain a directory if a remote + # type probe fails and the optional file download is attempted instead. + for relative in ("agent", "verifier", "artifacts/logs/artifacts"): + (session.directory / relative).mkdir(parents=True, exist_ok=True) + session.result = {"runtime": "gym-tb4-native", "runtime_version": NATIVE_VERSION, "started_at": now()} + ready = False + try: + await session.slots.acquire() + session.owns_slot = True + session.task = await loader.load(session.request.task_name, session.request.task_ref) + session.environment = Environment( + session.task, session.config.environment, session.session_id, session.directory + ) + # Construct and validate the verifier configuration before allocating + # either environment, but allocate its resources only after collection. + session.verifier_environment = Environment( + session.task, + session.config.environment, + session.session_id + "__verifier__trial", + session.directory, + verifier=True, + ) + if session.config.environment.efs_logs_host_path: + session.shared_logs = SharedLogs(session.environment) + session.environment.shared_logs = session.shared_logs + session.verifier_environment.shared_logs = session.shared_logs + # Reject mount conflicts before allocating the helper or workloads. + session.environment.build_spec() + session.verifier_environment.build_spec() + + async def provision(): + if session.shared_logs: + try: + await session.shared_logs.start() + except Exception as exc: + if "VOLUME::HOST_PATH_NOT_ALLOWED" not in str( + exc + ) or session.config.environment.efs_logs_host_path not in str(exc): + raise + await session.shared_logs.stop() + session.environment.shared_logs = None + session.verifier_environment.shared_logs = None + session.diagnostics.append({"operation": "efs_logs_fallback", "role": "helper", "error": str(exc)}) + session.persist() + await session.environment.start() + if getattr(session.environment, "efs_logs_fallback", None): + session.verifier_environment.shared_logs = None + session.diagnostics.append( + {"operation": "efs_logs_fallback", "role": "agent", "error": session.environment.efs_logs_fallback} + ) + + session.result["environment_setup"] = {"started_at": now()} + try: + await asyncio.wait_for(provision(), session.task.config.environment.build_timeout_sec) + except TimeoutError as exc: + exception(session, exc, "EnvironmentStartTimeoutError") + raise + finally: + session.result["environment_setup"]["finished_at"] = now() + await session.environment.healthcheck() + async with session.lock: + session.phase = "ready" + session.setup_deadline = monotonic() + SETUP_TIMEOUT_SEC + session.deadlines["setup_started_at"] = now() + session.result["agent_setup"] = {"started_at": now()} + session.watchdog = asyncio.create_task(watch_deadline(session, setup=True)) + session.persist() + # Like the reference, descriptor I/O is inside the setup allowance. + connection = await session.environment.main_connection() + async with session.lock: + if session.phase != "ready": + raise RuntimeError("Episode ended while preparing its descriptor") + session.seed = SandboxedSeedResponse( + session_id=session.session_id, + sandbox=connection, + instruction=session.task.instruction, + user=session.task.config.agent.user, + agent_timeout_sec=min( + session.task.config.agent.timeout_sec, session.config.agent_max_timeout_sec or float("inf") + ), + setup_timeout_sec=SETUP_TIMEOUT_SEC, + mcp_servers=[s.model_dump() for s in session.task.config.environment.mcp_servers], + skills_dir=session.task.config.environment.skills_dir, + ) + ready = True + session.persist() + except asyncio.CancelledError: + session.termination = session.termination or AgentTermination(reason="cancelled") + exception(session, "Preparation or harness setup cancelled", "CancelledError") + except Exception as exc: + exception(session, exc) + finally: + if not ready: + await cleanup(session) + + +async def start_session(session): + async with session.lock: + if session.finalization is not None: + raise HTTPException(409, "Episode is already finishing") + if session.phase == "ready" and session.seed: + if monotonic() >= session.setup_deadline: + finish = _request_finish( + session, AgentTermination(reason="timeout", detail="Harness setup deadline reached") + ) + else: + stop_watchdog(session) + session.result["agent_setup"]["finished_at"] = now() + session.result["agent_execution"] = {"started_at": now()} + session.deadlines["agent_started_at"] = now() + session.deadline = monotonic() + session.seed.agent_timeout_sec + session.phase = "agent_running" + session.watchdog = asyncio.create_task(watch_deadline(session)) + session.persist() + return {"agent_timeout_sec": max(0, session.deadline - monotonic())} + elif session.phase == "agent_running": + if monotonic() < session.deadline: + return {"agent_timeout_sec": max(0, session.deadline - monotonic())} + finish = _request_finish( + session, AgentTermination(reason="timeout", detail="Resources agent deadline reached") + ) + else: + raise HTTPException(409, "Episode is not ready or running") + await asyncio.shield(finish) + raise HTTPException(409, "Episode deadline reached") + + +async def watch_deadline(session, setup=False): + deadline = session.setup_deadline if setup else session.deadline + await asyncio.sleep(max(0, deadline - monotonic())) + detail = "Harness setup deadline reached" if setup else "Resources agent deadline reached" + # A watchdog only requests finalization; it never owns or awaits it. + await request_finish(session, AgentTermination(reason="timeout", detail=detail)) + + +def _request_finish(session, termination): + """Called only with the lock held. First accepted finish wins, except that + an already elapsed execution deadline always precedes a newly handled request. + """ + if session.finalization is not None or session.phase == "closed": + return session.finalization + grade = session.phase == "agent_running" + if grade and monotonic() >= session.deadline: + termination = AgentTermination( + reason="timeout", detail="Resources agent deadline reached", artifacts=termination.artifacts + ) + session.termination = session.termination or termination.model_copy(deep=True) + stop_watchdog(session) + if grade: + session.result["agent_execution"]["finished_at"] = now() + session.phase = "verifying" + elif termination.reason == "timeout": + exception(session, termination.detail or "Harness setup timed out", "AgentSetupTimeoutError") + session.finalization = asyncio.create_task(finalize_session(session, grade=grade)) + session.persist() + return session.finalization + + +async def request_finish(session, termination, body=None): + async with session.lock: + if body is not None: + if session.phase in {"preparing", "ready"}: + raise HTTPException(409, "Agent setup has not completed") + if session.verify_body is None: + session.verify_body = body.model_copy(deep=True) + session.directory.mkdir(parents=True, exist_ok=True) + (session.directory / "gym-agent.json").write_text(body.model_dump_json(indent=2)) + finish = _request_finish(session, termination) + # Late worker artifacts remain references, even after deadline grading. + if body is not None and session.termination is not None: + session.termination.artifacts = list( + dict.fromkeys(session.termination.artifacts + body.termination.artifacts) + ) + session.persist() + return finish + + +async def finalize_session(session, *, grade): + try: + if not grade: + if session.preparation and not session.preparation.done(): + session.preparation.cancel() + await session.preparation + if session.environment and session.environment.main and not session.environment.closed: + await session.environment.quiesce_agent(session.session_id) + return + session.subphase = "quiesce" + session.persist() + await session.environment.quiesce_agent(session.session_id) + if session.termination.reason != "completed": + error_type = ( + "AgentTimeoutError" if session.termination.reason == "timeout" else "NonZeroAgentExitCodeError" + ) + exception(session, session.termination.detail or session.termination.reason, error_type) + session.subphase = "collect" + session.persist() + try: + await download_dir(session.environment.main, "/logs/agent", session.directory / "agent") + except Exception as exc: + session.diagnostics.append({"operation": "agent_logs", "error": str(exc)}) + await collect(session.environment, session.directory / "artifacts", session.diagnostics) + try: + await session.environment.stop() + except Exception as exc: + exception(session, exc) + if session.shared_logs and session.environment.closed: + try: + await session.shared_logs.prepare_verifier() + session.diagnostics.append( + { + "operation": "efs_artifact_restore", + "snapshot_ready": bool(session.shared_logs.restored_archive), + } + ) + except Exception as exc: + session.diagnostics.append({"operation": "efs_artifact_restore", "error": str(exc)}) + session.subphase = "verifier_setup" + session.result["verifier"] = {"started_at": now()} + session.persist() + try: + # Compatibility: both startups use the task's environment build + # budget; verifier healthchecks are not run by the reference. + await asyncio.wait_for( + session.verifier_environment.start(), + session.task.config.environment.build_timeout_sec, + ) + if getattr(session.verifier_environment, "efs_logs_fallback", None): + session.diagnostics.append( + { + "operation": "efs_logs_fallback", + "role": "verifier", + "error": session.verifier_environment.efs_logs_fallback, + } + ) + await restore(session.verifier_environment, session.directory / "artifacts") + session.subphase = "verifier_execution" + session.persist() + session.result["verifier_result"] = await run_verifier( + session.verifier_environment, + session.directory, + session.diagnostics, + ) + finally: + session.result["verifier"]["finished_at"] = now() + except asyncio.CancelledError: + exception(session, "Resources shutdown interrupted evaluation", "CancelledError") + except Exception as exc: + exception(session, exc) + finally: + await cleanup(session) + + +async def shutdown(sessions, timeout): + finalizers = [] + for session in sessions: + finish = await request_finish(session, AgentTermination(reason="cancelled", detail="Resources shutdown")) + if finish: + finalizers.append(finish) + if finalizers: + _, pending = await asyncio.wait(finalizers, timeout=timeout) + for task in pending: + task.cancel() + await asyncio.gather(*finalizers, return_exceptions=True) diff --git a/resources_servers/terminal_bench_4/requirements.txt b/resources_servers/terminal_bench_4/requirements.txt new file mode 100644 index 0000000000..71be0c499e --- /dev/null +++ b/resources_servers/terminal_bench_4/requirements.txt @@ -0,0 +1,4 @@ +-e nemo-gym[dev,sandbox] @ ../../ +pathspec==1.1.1 +opensandbox==0.1.16 +httpx-aiohttp diff --git a/resources_servers/terminal_bench_4/shared_logs.py b/resources_servers/terminal_bench_4/shared_logs.py new file mode 100644 index 0000000000..11e890fdd2 --- /dev/null +++ b/resources_servers/terminal_bench_4/shared_logs.py @@ -0,0 +1,226 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Episode-owned EFS logs, with isolated roles and a collected artifact snapshot.""" + +import asyncio +import io +import json +import os +import shlex +import tarfile +from copy import deepcopy +from pathlib import PurePosixPath +from uuid import uuid4 + +from nemo_gym.sandbox import AsyncSandbox, SandboxResources, SandboxSpec, resolve_provider_config + + +class SharedLogs: + mount = "/tmp/tb4-efs" + + def __init__(self, environment): + config = environment.config + self.host_path = config.efs_logs_host_path + path = PurePosixPath(self.host_path) + if not path.is_absolute() or ".." in path.parts: + raise ValueError("EFS logs host path must be absolute without traversal") + self.relative = "tb4-logs-" + uuid4().hex + self.owner_token = uuid4().hex + self.session_id = environment.session_id + "__logs" + self.archive_name = ".tb4-artifacts-" + uuid4().hex + ".tar.gz" + self.archive_digest = None + self.archive_owner = None + self.restored_archive = None + self.closed = False + self.resources = [] + self.cleanup_errors = [] + self._cleanup_task = None + self.initialized = False + provider = deepcopy(environment.provider_config) + self.pool = environment.pool + # Storage management is CPU work, even for a GPU episode. Both pools + # must refer to the same EFS share on the cell. + if config.sandbox_split_endpoints: + self.pool = "cpu" + connection = provider["opensandbox"].setdefault("connection", {}) + connection.update( + domain=os.environ["OPENSANDBOX_DOMAIN_CPU"], api_key=os.environ["OPENSANDBOX_API_KEY_CPU"] + ) + metadata = dict(environment.build_spec().metadata) + metadata.update({"tb4-session": self.session_id, "tb4-role": "logs-helper"}) + if self.pool != "default": + metadata["nemo-gym.nvidia.com/resource-pool"] = self.pool + self.main = AsyncSandbox( + resolve_provider_config(provider), + SandboxSpec( + image=config.efs_logs_init_image, + resources=SandboxResources(cpu=1, memory_mib=512), + ttl_s=config.sandbox_ttl_s, + ready_timeout_s=config.sandbox_ready_timeout_s, + entrypoint=["sh", "-c", "while :; do sleep 3600; done"], + metadata=metadata, + provider_options={ + "resource_requests": "limits", + "volumes": [{"name": "tb4-efs", "host": {"path": self.host_path}, "mountPath": self.mount}], + }, + ), + ) + + @property + def root(self): + return f"{self.mount}/{self.relative}" + + def volume(self, role): + if role not in {"agent", "verifier"}: + raise ValueError("Unknown EFS log role") + return { + "name": "tb4-logs", + "host": {"path": self.host_path}, + "subPath": f"{self.relative}/{role}", + "mountPath": "/logs", + "readOnly": False, + } + + async def python(self, source, *args): + command = "python3 -c " + shlex.quote(source) + " " + " ".join(shlex.quote(str(a)) for a in args) + result = await self.main.exec(command, timeout_s=600) + if result.return_code: + raise RuntimeError(f"EFS logs operation failed: {result.stderr}") + return result + + async def start(self): + await self.main.start() + # Never adopt an existing episode directory. Roles can only access their + # own subPath; the shared root is mounted solely in the trusted helper. + self.initialized = True + await self.python( + "from pathlib import Path; import sys\n" + "root = Path(sys.argv[1]); root.mkdir(mode=0o755)\n" + "(root / '.owner').write_text(sys.argv[2])\n" + "for role in ('agent', 'verifier'):\n" + " p = root / role; p.mkdir(); p.chmod(0o777)\n", + self.root, + self.owner_token, + ) + + async def initialize_role(self, environment): + result = await environment.exec("id -u; id -g", timeout_sec=60) + try: + uid, gid = (int(value) for value in result.stdout.split()) + if result.return_code or min(uid, gid) < 0: + raise ValueError("Invalid workload identity") + except (TypeError, ValueError) as exc: + raise RuntimeError("Unable to determine the task log owner") from exc + # Give the mounted directory to the image's actual user. Keeping it + # world-writable would let a dropped verifier child rename protected + # /logs/verifier and replace its reward file despite chmod 700 there. + await self.python( + "import os, sys\n" + "os.chown(sys.argv[1], int(sys.argv[2]), int(sys.argv[3]))\n" + "os.chmod(sys.argv[1], 0o755)\n", + f"{self.root}/{environment.log_role}", + uid, + gid, + ) + + def collection_archive(self, artifact, artifacts): + """Optimize only the unambiguous conventional main log artifact. + + Other artifacts retain their ordered host restore, including overlaps, + same-source recollection, sidecars, and custom destinations. + """ + if artifact.source != "/logs/artifacts" or artifact.service not in (None, "main"): + return None + source, target = PurePosixPath(artifact.source), artifact.host_path + for other in artifacts: + if other is artifact: + continue + other_source, other_target = PurePosixPath(other.source), other.host_path + if ( + source == other_source + or source in other_source.parents + or other_source in source.parents + or target == other_target + or target in other_target.parents + or other_target in target.parents + ): + return None + return f"/logs/{self.archive_name}" + + def retain_archive(self, digest, directory): + self.archive_digest = digest + if digest: + # Match the ownership headers upload_dir would produce from the + # downloaded, data-filtered host snapshot. + with tarfile.open(fileobj=io.BytesIO(), mode="w") as tar: + info = tar.gettarinfo(str(directory), arcname=".") + self.archive_owner = [info.uid, info.gid, info.uname, info.gname] + + async def prepare_verifier(self): + if not self.archive_digest: + return + # The exact collected archive survives agent deletion on EFS. Validate + # its digest and apply the same data filter as download_dir before + # repacking, so exclusions, symlinks and permissions match host restore. + result = await self.python( + "import hashlib, json, pathlib, sys, tarfile, tempfile\n" + "root, name, digest, owner = sys.argv[1:]\n" + "root = pathlib.Path(root); source = root / 'agent' / name\n" + "if source.is_symlink() or not source.is_file(): sys.exit(0)\n" + "with source.open('rb') as f:\n" + " if hashlib.file_digest(f, 'sha256').hexdigest() != digest: sys.exit(0)\n" + "uid, gid, uname, gname = json.loads(owner)\n" + "def headers(info):\n" + " info.uid, info.gid, info.uname, info.gname = uid, gid, uname, gname\n" + " return info\n" + "with tempfile.TemporaryDirectory(dir=root) as tmp:\n" + " with tarfile.open(source, 'r:gz') as tar: tar.extractall(tmp, filter='data')\n" + " with tarfile.open(root / 'verifier' / name, 'w:gz') as tar:\n" + " tar.add(tmp, arcname='.', filter=headers)\n" + "print('ready')\n", + self.root, + self.archive_name, + self.archive_digest, + json.dumps(self.archive_owner), + ) + if result.stdout.strip() == "ready": + self.restored_archive = f"/logs/{self.archive_name}" + + def resource_identities(self): + result = [{"efs_host_path": self.host_path, "efs_subpath": self.relative}] + handle = getattr(self.main, "_handle", None) + if handle is not None: + result.append({"service": "logs-helper", "provider": self.pool, "sandbox_id": handle.sandbox_id}) + return result + + async def stop(self, *, remove_data=True): + if self._cleanup_task is None: + self._cleanup_task = asyncio.create_task(self._stop(remove_data)) + await asyncio.shield(self._cleanup_task) + + async def _stop(self, remove_data): + self.resources = self.resource_identities() + errors = [] + if self.initialized and remove_data: + try: + await self.python( + "import pathlib, shutil, sys\n" + "root = pathlib.Path(sys.argv[1])\n" + "assert not root.is_symlink()\n" + "if root.exists():\n" + " assert (root / '.owner').read_text() == sys.argv[2]\n" + " shutil.rmtree(root)\n", + self.root, + self.owner_token, + ) + except Exception as exc: + errors.append(exc) + try: + await self.main.stop() + except Exception as exc: + errors.append(exc) + if errors: + self.cleanup_errors.extend({"error": str(exc), "resources": self.resources} for exc in errors) + raise ExceptionGroup("EFS logs cleanup failed", errors) + self.closed = True diff --git a/resources_servers/terminal_bench_4/task.py b/resources_servers/terminal_bench_4/task.py new file mode 100644 index 0000000000..b7aad4cfb1 --- /dev/null +++ b/resources_servers/terminal_bench_4/task.py @@ -0,0 +1,320 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The execution settings and content-addressed packages used by pinned TB4.""" + +import asyncio +import hashlib +import os +import re +import tarfile +import tempfile +import tomllib +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Any, Literal +from urllib.parse import quote + +import aiohttp +import pathspec +from pydantic import BaseModel, ConfigDict, Field, model_validator + +from nemo_gym.server_utils import request + + +class Settings(BaseModel): + model_config = ConfigDict(extra="forbid", allow_inf_nan=False) + + +class Healthcheck(Settings): + command: str + interval_sec: float = Field(default=5, ge=0) + timeout_sec: float = Field(default=30, gt=0) + start_period_sec: float = Field(default=0, ge=0) + start_interval_sec: float = Field(default=5, ge=0) + retries: int = Field(default=3, gt=0) + + +class MCPServer(Settings): + name: str + transport: Literal["stdio", "sse", "streamable-http"] = "sse" + url: str | None = None + command: str | None = None + args: list[str] = Field(default_factory=list) + + @model_validator(mode="before") + @classmethod + def normalize(cls, data): + if isinstance(data, dict) and data.get("transport") == "http": + data = {**data, "transport": "streamable-http"} + return data + + @model_validator(mode="after") + def validate_transport(self): + if not (self.command if self.transport == "stdio" else self.url): + raise ValueError("MCP transport requires command or URL") + return self + + +class PhaseSettings(Settings): + network_mode: Literal["public", "no-network"] | None = None + allowed_hosts: list[str] | None = None + + @model_validator(mode="after") + def no_allowlist(self): + if self.allowed_hosts is not None: + raise ValueError("TB4 does not support network allowlists") + return self + + +class EnvironmentSettings(PhaseSettings): + docker_image: str + build_timeout_sec: float = Field(default=600, gt=0) + os: Literal["linux"] = "linux" + cpus: int | None = Field(default=None, gt=0) + memory_mb: int | None = Field(default=None, gt=0) + storage_mb: int | None = Field(default=None, gt=0) + gpus: int | None = Field(default=None, ge=0) + gpu_types: list[str] | None = Field(default=None, max_length=1) + env: dict[str, str] = Field(default_factory=dict) + mcp_servers: list[MCPServer] = Field(default_factory=list) + skills_dir: str | None = None + healthcheck: Healthcheck | None = None + workdir: str | None = None + network_mode: Literal["public", "no-network"] = "public" + + @model_validator(mode="before") + @classmethod + def legacy_fields(cls, data): + data = dict(data) + allow = data.pop("allow_internet", None) + if allow is not None: + data.setdefault("network_mode", "public" if allow else "no-network") + for old, new in (("memory", "memory_mb"), ("storage", "storage_mb")): + if old in data: + raw = data.pop(old).strip().upper() + size = int(float(raw[:-1]) * {"G": 1024, "M": 1, "K": 1 / 1024}[raw[-1]]) + if new in data and data[new] != size: + raise ValueError(f"Conflicting {old} and {new}") + data[new] = size + return data + + +class AgentSettings(PhaseSettings): + timeout_sec: float = Field(default=28800, gt=0) + user: str | int | None = None + + +class CollectHook(Settings): + command: str + service: str = "main" + timeout_sec: float = Field(default=60, gt=0) + user: str | int | None = None + + +class VerifierSettings(PhaseSettings): + timeout_sec: float = Field(default=600, gt=0) + user: str | int | None = None + env: dict[str, str] = Field(default_factory=dict) + environment_mode: Literal["separate", "shared"] | None = None + environment: EnvironmentSettings | None = None + collect: list[CollectHook] = Field(default_factory=list) + + +class Artifact(Settings): + source: str + destination: str | None = None + service: str | None = None + exclude: list[str] = Field(default_factory=list) + + @model_validator(mode="after") + def validate_paths(self): + for path in (self.source, self.destination): + if path and (".." in PurePosixPath(path).parts or "\\" in path): + raise ValueError("Artifact paths must be contained POSIX paths") + if self.destination and ( + PurePosixPath(self.destination).is_absolute() + or PurePosixPath(self.destination) in (PurePosixPath("."), PurePosixPath("manifest.json")) + ): + raise ValueError("Artifact destination must be relative and cannot shadow the manifest") + if self.service not in (None, "main") and not self.source.startswith("/"): + raise ValueError("Sidecar artifacts require absolute source paths") + return self + + @property + def host_path(self): + return Path(self.destination or self.source.lstrip("/")) + + +class TaskSettings(Settings): + schema_version: str = "1.4" + task: dict[str, Any] = Field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) + source: str | None = None + solution: dict[str, Any] = Field(default_factory=dict) + agent: AgentSettings = Field(default_factory=AgentSettings) + environment: EnvironmentSettings + verifier: VerifierSettings = Field(default_factory=VerifierSettings) + artifacts: list[Artifact] = Field(default_factory=list) + + @model_validator(mode="before") + @classmethod + def normalize(cls, data): + data = dict(data) + if data.pop("steps", None) or data.pop("multi_step_reward_strategy", None): + raise ValueError("TB4 requires single-step task packages") + if "version" in data: + data.setdefault("schema_version", data.pop("version")) + data["artifacts"] = [{"source": a} if isinstance(a, str) else a for a in data.get("artifacts", [])] + return data + + @model_validator(mode="after") + def validate_execution(self): + if self.verifier.environment_mode == "shared" and self.verifier.environment is not None: + raise ValueError("Shared verification cannot define a separate environment") + # All 66 pinned packages use separate verification. Reject unobserved + # shared mode rather than silently changing collection or test upload. + if self.verifier.environment_mode != "separate" and self.verifier.environment is None: + raise ValueError("The pinned TB4 profile requires separate verification") + for phase, env in ((self.agent, self.environment), (self.verifier, self.verifier_environment)): + if phase.network_mode is not None and phase.network_mode != env.network_mode: + raise ValueError("Dynamic network policy transitions are unsupported") + return self + + @property + def verifier_environment(self): + return self.verifier.environment or self.environment.model_copy(deep=True) + + @property + def collected_artifacts(self): + entries = list(self.artifacts) + if not any(a.source.rstrip("/") == "/logs/artifacts" and a.service in (None, "main") for a in entries): + entries.insert(0, Artifact(source="/logs/artifacts")) + return entries + + +@dataclass +class Task: + path: Path + name: str + ref: str + config: TaskSettings + instruction: str + + @classmethod + def read(cls, path, name, ref): + path = Path(path).resolve() + for entry in path.rglob("*"): + if entry.is_symlink() and not entry.resolve().is_relative_to(path): + raise ValueError(f"Package link escapes task: {entry.relative_to(path)}") + config = TaskSettings.model_validate(tomllib.loads((path / "task.toml").read_text())) + if config.task.get("name", name) != name or not (path / "environment").is_dir(): + raise ValueError("Package identity or environment layout is invalid") + lines = (path / "instruction.md").read_text().split("\n") + while lines and re.match(r"^(|#.*canary.*)$", lines[0].strip(), re.IGNORECASE): + lines.pop(0) + while lines and not lines[0].strip(): + lines.pop(0) + return cls(path, name, ref, config, "\n".join(lines)) + + +def content_hash(path: Path) -> str: + """SHA256 of sorted `relative_path + NUL + file_sha256_hex + LF` records. + + Matches the package publisher's file set, not dirhash or archive bytes. + A package .gitignore replaces the publisher's default ignore patterns. + """ + singles = [path / name for name in ("task.toml", "instruction.md", "README.md", "trajectory.json")] + files = [p for p in singles if p.is_file()] + for name in ("environment", "tests", "solution", "steps"): + files.extend(p for p in (path / name).rglob("*") if p.is_file()) + ignores = ["__pycache__/", "*.pyc", ".DS_Store", "*.swp", "*.swo", "*~"] + if (path / ".gitignore").exists(): + ignores = (path / ".gitignore").read_text().splitlines() + spec = pathspec.PathSpec.from_lines("gitignore", ignores) + digest = hashlib.sha256() + for file in sorted(files, key=lambda p: p.relative_to(path).as_posix()): + relative = file.relative_to(path).as_posix() + if spec.match_file(relative): + continue + if not file.resolve().is_relative_to(path.resolve()): + raise ValueError(f"Package file escapes task: {relative}") + with file.open("rb") as stream: + file_hash = hashlib.file_digest(stream, "sha256").hexdigest() + digest.update(f"{relative}\0{file_hash}\n".encode()) + return digest.hexdigest() + + +def resolve_env(values): + def substitute(match): + name, default = match.group(1), match.group(2) + if name in os.environ: + return os.environ[name] + if default is not None: + return default + raise ValueError(f"Missing task environment variable: {name}") + + pattern = re.compile(r"\$\{([^}:]+)(?::-(.*))?\}") + return {k: substitute(match) if (match := pattern.fullmatch(v)) else v for k, v in values.items()} + + +class PackageLoader: + def __init__(self, download_dir=None): + self.root = Path(download_dir or Path.home() / ".cache/harbor/tasks/packages") + self._locks = {} + + async def load(self, name, ref): + if not re.fullmatch(r"terminal-bench/[a-z0-9][a-z0-9-]*", name) or not re.fullmatch( + r"sha256:[a-f0-9]{64}", ref + ): + raise ValueError("A trusted task name and SHA256 pin are required") + target = self.root / name / ref[7:] + async with self._locks.setdefault((name, ref), asyncio.Lock()): + if not target.exists(): + await self._download(name, ref, target) + actual = await asyncio.to_thread(content_hash, target) + if actual != ref[7:]: + raise ValueError(f"Package content hash mismatch for {name}: expected {ref}, got sha256:{actual}") + return await asyncio.to_thread(Task.read, target, name, ref) + + async def _download(self, name, ref, target): + # Public registry credentials identify the public project, not a user. + base = "https://ofhuhcpkvzjlejydnvyd.supabase.co" + headers = {"apikey": "sb_publishable_Z-vuQbpvpG-PStjbh4yE0Q_e-d3MTIH"} + org, short_name = name.split("/") + async with await request( + "POST", + base + "/rest/v1/rpc/resolve_task_version", + headers=headers, + json={"p_org": org, "p_name": short_name, "p_ref": ref}, + timeout=aiohttp.ClientTimeout(total=120), + ) as response: + response.raise_for_status() + resolved = await response.json() + if not resolved or resolved["content_hash"].removeprefix("sha256:") != ref[7:]: + raise ValueError("Registry returned a different package identity") + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=".tb4-download-", dir=target.parent) as tmp: + archive = Path(tmp) / "task.tar.gz" + url = base + "/storage/v1/object/packages/" + quote(resolved["archive_path"], safe="/") + async with await request( + "GET", url, headers=headers, timeout=aiohttp.ClientTimeout(total=120) + ) as response: + response.raise_for_status() + with archive.open("wb") as stream: + async for chunk in response.content.iter_chunked(1024 * 1024): + stream.write(chunk) + staged = Path(tmp) / "package" + staged.mkdir() + with tarfile.open(archive, "r:gz") as tar: + tar.extractall(staged, filter="data") + Task.read(staged, name, ref) + if await asyncio.to_thread(content_hash, staged) != ref[7:]: + raise ValueError("Downloaded package content hash mismatch") + # A second process may have promoted the same immutable pin. + try: + staged.rename(target) + except OSError: + if not target.is_dir(): + raise + # load() validates the winning process's package before use. diff --git a/resources_servers/terminal_bench_4/tests/test_agent_lifecycle.py b/resources_servers/terminal_bench_4/tests/test_agent_lifecycle.py new file mode 100644 index 0000000000..8f7adb85ea --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_agent_lifecycle.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nemo_gym.base_resources_server import BaseRunRequest +from nemo_gym.sandbox import agent as lifecycle +from nemo_gym.sandbox.handoff import AgentTermination +from nemo_gym.server_utils import SESSION_ID_KEY + + +@pytest.mark.parametrize("reason", ["completed", "timeout", "nonzero_exit", "infrastructure_error", "cancelled"]) +async def test_borrowed_agent_always_verifies_and_releases(monkeypatch, reason): + body = BaseRunRequest(responses_create_params={"input": []}) + sandbox = MagicMock(release=AsyncMock()) + connect = AsyncMock(return_value=sandbox) + monkeypatch.setattr(lifecycle.AsyncSandbox, "connect", connect) + monkeypatch.setattr(lifecycle, "create_provider", lambda _: MagicMock()) + monkeypatch.setattr(lifecycle, "get_global_config_dict", lambda: {}) + monkeypatch.setattr(lifecycle, "raise_for_status", AsyncMock()) + seed = { + "session_id": "test", + "sandbox": {"provider": "cpu", "sandbox_id": "remote"}, + "instruction": "Official instruction", + "agent_timeout_sec": 10, + "setup_timeout_sec": 5, + } + monkeypatch.setattr( + lifecycle, "get_response_json", AsyncMock(side_effect=[seed, {"agent_timeout_sec": 7}, {"reward": 1}]) + ) + post = AsyncMock(return_value=SimpleNamespace(cookies={"resources-session": "cookie"})) + agent = SimpleNamespace( + config=SimpleNamespace( + name="test", + resources_server=SimpleNamespace(name="resources"), + model_server=SimpleNamespace(name="model"), + sandbox_providers={"cpu": {"local": {}}}, + ), + server_client=SimpleNamespace(post=post), + rollout_id_from_run=lambda _: "rollout", + ) + + async def execute(sandbox, seed, budget): + assert budget == 7 + if reason == "timeout": + raise TimeoutError() + if reason == "cancelled": + raise asyncio.CancelledError() + if reason == "infrastructure_error": + raise ConnectionError("model unavailable") + return lifecycle.empty_response(body.responses_create_params, "model"), AgentTermination(reason=reason), {} + + run = lifecycle.run_borrowed( + agent, + SimpleNamespace(cookies={"incoming": "cookie"}, session={SESSION_ID_KEY: "client"}), + body, + setup=AsyncMock(), + execute=execute, + ) + if reason == "cancelled": + with pytest.raises(asyncio.CancelledError): + await run + else: + assert (await run)["reward"] == 1 + assert body.responses_create_params.input == [] + assert connect.await_args.kwargs["owns_sandbox"] is False + assert [call.kwargs["url_path"] for call in post.await_args_list] == ["/seed_session", "/start_session", "/verify"] + verify = post.await_args_list[-1].kwargs + assert verify["cookies"] == {"incoming": "cookie", "resources-session": "cookie"} + assert verify["json"]["termination"]["reason"] == reason + assert verify["json"]["session_id"] == "test" + sandbox.release.assert_awaited_once() + + +@pytest.mark.parametrize("failure", ["alias", "connect", "setup", "cancel"]) +async def test_failed_attachment_or_setup_cancels_owned_session(tmp_path, monkeypatch, failure): + monkeypatch.chdir(tmp_path) + sandbox = MagicMock(release=AsyncMock()) + provider = MagicMock(aclose=AsyncMock()) + connect = AsyncMock( + side_effect=ConnectionError("attach failed") if failure == "connect" else None, return_value=sandbox + ) + monkeypatch.setattr(lifecycle.AsyncSandbox, "connect", connect) + monkeypatch.setattr(lifecycle, "create_provider", lambda _: provider) + monkeypatch.setattr(lifecycle, "get_global_config_dict", lambda: {}) + monkeypatch.setattr(lifecycle, "raise_for_status", AsyncMock()) + seed = { + "session_id": "test", + "sandbox": {"provider": "cpu", "sandbox_id": "box"}, + "instruction": "task", + "agent_timeout_sec": 1, + "setup_timeout_sec": 1, + } + monkeypatch.setattr(lifecycle, "get_response_json", AsyncMock(side_effect=[seed, {}])) + post = AsyncMock(return_value=SimpleNamespace(cookies={})) + agent = SimpleNamespace( + config=SimpleNamespace( + name="test", + resources_server=SimpleNamespace(name="resources"), + model_server=SimpleNamespace(name="model"), + sandbox_providers={} if failure == "alias" else {"cpu": {"local": {}}}, + ), + server_client=SimpleNamespace(post=post), + rollout_id_from_run=lambda _: None, + ) + setup = AsyncMock(side_effect=asyncio.CancelledError() if failure == "cancel" else RuntimeError("setup failed")) + execute = AsyncMock() + expected = { + "alias": ValueError, + "connect": ConnectionError, + "setup": RuntimeError, + "cancel": asyncio.CancelledError, + }[failure] + with pytest.raises(expected): + await lifecycle.run_borrowed( + agent, + SimpleNamespace(cookies={}, session={SESSION_ID_KEY: "client"}), + BaseRunRequest(responses_create_params={"input": []}), + setup=setup, + execute=execute, + ) + assert [c.kwargs["url_path"] for c in post.await_args_list] == ["/seed_session", "/cancel_session"] + execute.assert_not_awaited() + if failure == "connect": + provider.aclose.assert_awaited_once() + if failure in {"setup", "cancel"}: + sandbox.release.assert_awaited_once() diff --git a/resources_servers/terminal_bench_4/tests/test_benchmark_preparation.py b/resources_servers/terminal_bench_4/tests/test_benchmark_preparation.py new file mode 100644 index 0000000000..6d8dbededc --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_benchmark_preparation.py @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json + +import pytest + +from benchmarks.terminal_bench_4 import prepare as preparation + + +def test_prepared_names_match_pinned_manifest(tmp_path, monkeypatch): + monkeypatch.setattr(preparation, "OUTPUT_PATH", tmp_path / "benchmark.jsonl") + output = preparation.prepare() + rows = [json.loads(line) for line in output.read_text().splitlines()] + manifest = json.loads((preparation.BENCHMARK_DIR / "manifest.json").read_text()) + tasks = {"terminal-bench/" + task["name"]: task["ref"] for task in manifest["tasks"]} + assert len(rows) == len(tasks) == 66 + assert {row["task_name"] for row in rows} == tasks.keys() + for row in rows: + assert row["task_ref"] == tasks[row["task_name"]] + assert row["dataset_ref"] == manifest["ref"] + assert "path" not in row + + +@pytest.mark.parametrize("category,count", [("cpu", 52), ("compose", 11), ("gpu", 3)]) +def test_category_selections(tmp_path, monkeypatch, category, count): + monkeypatch.setattr(preparation, "OUTPUT_PATH", tmp_path / "benchmark.jsonl") + assert len(preparation.prepare(category=category).read_text().splitlines()) == count + + +def test_explicit_names_and_invalid_selections(tmp_path, monkeypatch): + monkeypatch.setattr(preparation, "OUTPUT_PATH", tmp_path / "benchmark.jsonl") + row = json.loads(preparation.prepare(task_names=["ks-solver-cpp"]).read_text()) + assert row["task_name"] == "terminal-bench/ks-solver-cpp" + with pytest.raises(ValueError, match="Unknown TB4 task"): + preparation.prepare(task_names=["missing"]) + with pytest.raises(ValueError, match="Unknown TB4 category"): + preparation.prepare(category="missing") + with pytest.raises(ValueError, match="empty"): + preparation.prepare(task_names=[]) diff --git a/resources_servers/terminal_bench_4/tests/test_collection_verifier.py b/resources_servers/terminal_bench_4/tests/test_collection_verifier.py new file mode 100644 index 0000000000..f3072b131a --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_collection_verifier.py @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +import re +import shutil +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from resources_servers.terminal_bench_4.collection import collect +from resources_servers.terminal_bench_4.task import TaskSettings +from resources_servers.terminal_bench_4.verifier import ( + RewardFileEmptyError, + RewardFileNotFoundError, + VerifierOutputParseError, + VerifierTimeoutError, + parse_reward, + restore, + run_verifier, +) + + +class FilesystemSandbox: + """Exercise real tar and shell transfers against isolated temporary trees.""" + + def __init__(self, root): + self.root = root + self.commands = [] + for name in ["app", "logs/artifacts", "logs/agent", "logs/verifier", "tests", "tmp", "evidence"]: + (root / name).mkdir(parents=True, exist_ok=True) + + def path(self, path): + return self.root / path.lstrip("/") + + async def exec(self, command, **kwargs): + self.commands.append((command, kwargs)) + # Only replace the controlled virtual container roots used by fixtures. + translated = re.sub( + r"/(app|logs|tests|tmp|evidence)(?=/|\s|[\x27\x22;]|$)", lambda m: str(self.root / m[1]), command + ) + process = await asyncio.create_subprocess_shell( + translated, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await process.communicate() + return SimpleNamespace( + return_code=process.returncode, + stdout=stdout.decode(errors="replace"), + stderr=stderr.decode(errors="replace"), + ) + + async def upload(self, source, target): + self.path(target).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, self.path(target)) + + async def download(self, source, target): + shutil.copy2(self.path(source), target) + + +def environment(tmp_path, cfg): + main = FilesystemSandbox(tmp_path / "main") + sidecar = FilesystemSandbox(tmp_path / "sidecar") + env = SimpleNamespace(main=main, task=SimpleNamespace(config=cfg), stopped=False, stop_main=None) + + def sandbox(service=None): + return main if service in (None, "main") else sidecar + + async def execute(command, service=None, timeout_sec=None, user=None, env=None): + if service not in (None, "main"): + assert instance.stopped, "sidecar evidence was collected before main stopped" + return await sandbox(service).exec(command, timeout_s=timeout_sec, user=user, env=env) + + async def stop_main(): + instance.stopped = True + + instance = env + env.sandbox = sandbox + env.exec = execute + env.stop_main = stop_main + return env, main, sidecar + + +async def test_known_positive_main_sidecar_and_conventional_artifacts_restore(tmp_path): + cfg = TaskSettings.model_validate( + { + "environment": {"docker_image": "agent"}, + "verifier": { + "environment": {"docker_image": "verifier"}, + "collect": [ + {"command": "echo main-hook > /app/output/hook.txt"}, + {"service": "db", "command": "echo sidecar-hook > /evidence/hook.txt"}, + {"service": "db", "command": "exit 9"}, + ], + }, + "artifacts": [ + {"source": "/app/output", "destination": "saved", "exclude": ["*.tmp"]}, + {"source": "/evidence", "service": "db"}, + "/app/optional.dat", + ], + } + ) + env, main, sidecar = environment(tmp_path / "agent", cfg) + main.path("/app/output").mkdir() + main.path("/app/output/answer.txt").write_text("42") + main.path("/app/output/ignored.tmp").write_text("exclude me") + main.path("/logs/artifacts/report.txt").write_text("conventional") + sidecar.path("/evidence/state.txt").write_text("trusted-sidecar") + diagnostics = [] + manifest = await collect(env, tmp_path / "artifacts", diagnostics) + assert (tmp_path / "artifacts/saved/answer.txt").read_text() == "42" + assert not (tmp_path / "artifacts/saved/ignored.tmp").exists() + assert (tmp_path / "artifacts/evidence/state.txt").read_text() == "trusted-sidecar" + assert any(x["status"] == "failed" and x["source"] == "/app/optional.dat" for x in manifest) + assert any(x.get("return_code") == 9 for x in diagnostics) + verifier, target, _ = environment(tmp_path / "verifier", cfg) + target.path("/app/output").mkdir() + target.path("/app/output/stale.txt").write_text("must be removed") + await restore(verifier, tmp_path / "artifacts") + assert target.path("/app/output/answer.txt").read_text() == "42" + assert target.path("/app/output/hook.txt").read_text().strip() == "main-hook" + assert target.path("/evidence/hook.txt").read_text().strip() == "sidecar-hook" + assert target.path("/logs/artifacts/report.txt").read_text() == "conventional" + assert not target.path("/app/output/stale.txt").exists() + assert not target.path("/saved").exists() + # Baked-in tests validate bytes from every transfer channel. The test file + # is immutable input; native verification must never upload task-side tests. + script = target.path("/tests/test.sh") + script.write_text( + "#!/bin/sh\n" + + "\n".join( + [ + f'test "$(cat {target.path("/app/output/answer.txt")})" = 42 &&', + f'test "$(cat {target.path("/evidence/state.txt")})" = trusted-sidecar &&', + f'test "$(cat {target.path("/logs/artifacts/report.txt")})" = conventional &&', + f"echo 1 > {target.path('/logs/verifier/reward.txt')}", + ] + ) + + "\n" + ) + result = await run_verifier(verifier, tmp_path / "result", diagnostics) + assert result == {"rewards": {"reward": 1.0}} + assert not list(main.root.glob("tmp/.nemo-gym-*")) + assert not list(target.root.glob("tmp/.nemo-gym-*")) + + +async def test_failed_convention_probe_preserves_directory_for_restore(tmp_path): + cfg = TaskSettings.model_validate( + {"environment": {"docker_image": "agent"}, "verifier": {"environment": {"docker_image": "verifier"}}} + ) + env, main, _ = environment(tmp_path / "agent", cfg) + artifacts = tmp_path / "artifacts" + # This is the host layout created by prepare_session, matching the reference. + (artifacts / "logs/artifacts").mkdir(parents=True) + execute = env.exec + + async def failed_probe(command, **kwargs): + if command == "test -d /logs/artifacts": + return SimpleNamespace(return_code=1, stderr="operation not permitted") + return await execute(command, **kwargs) + + async def sdk_file_download(source, target): + # The provider's file API returns empty bytes when asked for a directory. + target.write_bytes(b"") + + env.exec = failed_probe + main.download = sdk_file_download + manifest = await collect(env, artifacts, []) + assert manifest[0]["status"] == "failed" + assert (artifacts / "logs/artifacts").is_dir() + verifier, target, _ = environment(tmp_path / "verifier", cfg) + await restore(verifier, artifacts) + assert target.path("/logs/artifacts").is_dir() + + +async def test_colliding_host_destinations_keep_first(tmp_path): + cfg = TaskSettings.model_validate( + { + "environment": {"docker_image": "agent"}, + "verifier": {"environment": {"docker_image": "verifier"}}, + "artifacts": [ + {"source": "/app/one.txt", "destination": "same.txt"}, + {"source": "/app/two.txt", "destination": "same.txt"}, + ], + } + ) + env, main, _ = environment(tmp_path / "agent", cfg) + main.path("/app/one.txt").write_text("first") + main.path("/app/two.txt").write_text("second") + records = await collect(env, tmp_path / "artifacts", []) + assert (tmp_path / "artifacts/same.txt").read_text() == "first" + assert records[-1]["status"] == "skipped" + + +@pytest.mark.parametrize("text,expected", [("0", 0), ("1", 1), ("0.5", 0.5), ("-1", -1)]) +def test_text_reward(tmp_path, text, expected): + (tmp_path / "reward.txt").write_text(text) + assert parse_reward(tmp_path) == {"reward": expected} + + +@pytest.mark.parametrize("text", ["NaN", "Infinity", "not a number"]) +def test_malformed_text_reward(tmp_path, text): + (tmp_path / "reward.txt").write_text(text) + with pytest.raises(VerifierOutputParseError): + parse_reward(tmp_path) + + +@pytest.mark.parametrize("text", ['{"reward":NaN}', '{"reward":1e309}', '{"reward":"1"}', "[1]", "1", "{broken"]) +def test_malformed_json_has_precedence_over_valid_text(tmp_path, text): + (tmp_path / "reward.txt").write_text("1") + (tmp_path / "reward.json").write_text(text) + with pytest.raises(VerifierOutputParseError): + parse_reward(tmp_path) + + +def test_zero_json_precedence_empty_and_missing(tmp_path): + with pytest.raises(RewardFileNotFoundError): + parse_reward(tmp_path) + (tmp_path / "reward.txt").write_text("1") + (tmp_path / "reward.json").write_text("") + with pytest.raises(RewardFileEmptyError): + parse_reward(tmp_path) + (tmp_path / "reward.json").write_text(json.dumps({"reward": 0})) + assert parse_reward(tmp_path) == {"reward": 0} + + +async def test_verifier_nonzero_exit_with_official_zero_is_completed(tmp_path): + cfg = TaskSettings.model_validate( + {"environment": {"docker_image": "a"}, "verifier": {"environment": {"docker_image": "v"}}} + ) + env, main, _ = environment(tmp_path / "verifier", cfg) + main.path("/tests/test.sh").write_text(f"#!/bin/sh\necho 0 > {main.path('/logs/verifier/reward.txt')}\nexit 4\n") + diagnostics = [] + assert await run_verifier(env, tmp_path / "result", diagnostics) == {"rewards": {"reward": 0}} + assert diagnostics[0]["return_code"] == 4 + + +async def test_verifier_timeout_does_not_promote_partial_reward(tmp_path, monkeypatch): + cfg = TaskSettings.model_validate( + {"environment": {"docker_image": "a"}, "verifier": {"timeout_sec": 0.01, "environment": {"docker_image": "v"}}} + ) + + async def execute(*args, **kwargs): + await asyncio.Event().wait() + + env = SimpleNamespace(task=SimpleNamespace(config=cfg), exec=execute, main=None) + from resources_servers.terminal_bench_4 import verifier + + download = AsyncMock() + monkeypatch.setattr(verifier, "download_dir", download) + with pytest.raises(VerifierTimeoutError): + await run_verifier(env, tmp_path, []) + download.assert_awaited_once() + + +async def test_transfer_fallbacks_preserve_file_bytes_and_empty_directories(tmp_path): + from resources_servers.terminal_bench_4.transfers import download_dir, upload_dir + + box = FilesystemSandbox(tmp_path / "sandbox") + local = tmp_path / "source" + (local / "empty").mkdir(parents=True) + (local / "file.txt").write_bytes(b"\x00payload\xff") + execute = box.exec + + async def without_tar(command, **kwargs): + if "tar -xzf" in command or command.startswith("tar -czf"): + return SimpleNamespace(return_code=1, stdout="", stderr="tar unavailable") + result = await execute(command, **kwargs) + if command.startswith("find "): + result.stdout = result.stdout.replace(str(box.root), "") + return result + + box.exec = without_tar + await upload_dir(box, local, "/app/output") + assert box.path("/app/output/file.txt").read_bytes() == b"\x00payload\xff" + assert box.path("/app/output/empty").is_dir() + await download_dir(box, "/app/output", tmp_path / "copied") + assert (tmp_path / "copied/file.txt").read_bytes() == b"\x00payload\xff" + with pytest.raises(RuntimeError, match="archive"): + await download_dir(box, "/app/output", tmp_path / "excluded", exclude=["*.tmp"]) + with pytest.raises(RuntimeError, match="list"): + await download_dir(box, "/app/missing", tmp_path / "missing") + + +async def test_restore_file_and_best_effort_collection_errors(tmp_path): + cfg = TaskSettings.model_validate( + { + "environment": {"docker_image": "a"}, + "verifier": {"environment": {"docker_image": "v"}, "collect": [{"command": "failing-hook"}]}, + "artifacts": ["/app/output.txt", {"source": "/evidence/state.txt", "service": "db"}], + } + ) + env, main, sidecar = environment(tmp_path / "agent", cfg) + main.path("/app/output.txt").write_text("file-data") + sidecar.path("/evidence/state.txt").write_text("sidecar-data") + original = env.exec + + async def execute(command, **kwargs): + if command == "failing-hook" or command.startswith("test -d /app"): + raise RuntimeError("optional hook/probe failed") + return await original(command, **kwargs) + + async def stop(): + env.stopped = True + raise RuntimeError("main stop failed") + + env.exec = execute + env.stop_main = stop + diagnostics = [] + records = await collect(env, tmp_path / "artifacts", diagnostics) + assert records[1]["type"] == "file" and records[1]["status"] == "ok" + assert {d["operation"] for d in diagnostics} == {"collect_hook", "stop_main"} + verifier, box, _ = environment(tmp_path / "verifier", cfg) + await restore(verifier, tmp_path / "artifacts") + assert box.path("/app/output.txt").read_text() == "file-data" + assert box.path("/evidence/state.txt").read_text() == "sidecar-data" diff --git a/resources_servers/terminal_bench_4/tests/test_compose_config.py b/resources_servers/terminal_bench_4/tests/test_compose_config.py new file mode 100644 index 0000000000..886f73bca4 --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_compose_config.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from copy import deepcopy + +import pytest + +from resources_servers.terminal_bench_4.compose_config import _bytes, resolve_compose + + +def image_record(**config): + return {"image": "repo@sha256:digest", "os": "linux", "architecture": "amd64", "config": config} + + +def test_harbor_main_command_image_startup_and_compose_literals(): + source = { + "services": { + "main": {"depends_on": ["peer"]}, + "peer": { + "image": "peer", + "environment": ["URL=http://main:8000", "VALUE=a=b"], + "shm_size": "1gb", + "mem_limit": 4096, + "healthcheck": {"test": ["CMD-SHELL", "echo $$HOME; test $? -eq 0"]}, + }, + } + } + original = deepcopy(source) + records = { + "main": image_record(Entrypoint=["/entrypoint.sh"], Cmd=["python3"], WorkingDir="/app"), + "peer": image_record(Cmd=["serve"], User="appuser", ExposedPorts={"8000/tcp": {}}), + } + result = resolve_compose(source, "main", records) + main, peer = result["services"]["main"], result["services"]["peer"] + assert main["entrypoint"] == ["/entrypoint.sh"] + assert main["command"] == ["sh", "-c", "sleep infinity"] + assert main["working_dir"] == "/app" + assert main["depends_on"] == {"peer": {"condition": "service_started"}} + assert peer["command"] == ["serve"] and peer["user"] == "appuser" + assert peer["expose"] == ["8000/tcp"] + assert peer["environment"] == {"URL": "http://main:8000", "VALUE": "a=b"} + assert peer["healthcheck"]["test"][1] == "echo $HOME; test $? -eq 0" + assert (peer["shm_size"], peer["mem_limit"]) == (1024**3, 4096) + assert source == original + + +def test_explicit_entrypoint_does_not_inherit_image_cmd_and_healthcheck_defaults(): + records = { + "main": image_record(), + "peer": image_record( + Cmd=["wrong"], + Healthcheck={ + "Test": ["CMD", "true"], + "Interval": 1_000_000_000, + "Timeout": 2_000_000_000, + "StartPeriod": 3_000_000_000, + "Retries": 4, + }, + ), + } + result = resolve_compose({"services": {"peer": {"image": "peer", "entrypoint": ["right"]}}}, "main", records) + peer = result["services"]["peer"] + assert peer["command"] == [] and peer["entrypoint"] == ["right"] + assert peer["healthcheck"] == { + "test": ["CMD", "true"], + "interval": "1.0s", + "timeout": "2.0s", + "start_period": "3.0s", + "retries": 4, + } + + +@pytest.mark.parametrize( + "source,records,match", + [ + (None, {}, "services mapping"), + ({"services": {}}, {}, "no recorded OCI"), + ({"services": {"main": {"environment": ["MISSING"]}}}, {"main": image_record()}, "unresolved"), + ({"services": {"main": {"command": "echo $UNRESOLVED"}}}, {"main": image_record()}, "substitutions"), + ({"services": {}}, {"main": {**image_record(), "architecture": "arm64"}}, "Linux/amd64"), + ], +) +def test_invalid_or_unresolved_compose_fails_before_provisioning(source, records, match): + with pytest.raises(ValueError, match=match): + resolve_compose(source, "main", records) + + +def test_memory_units(): + assert _bytes("1.5GiB") == 1610612736 + assert _bytes("512") == 512 + with pytest.raises(ValueError, match="memory size"): + _bytes("unlimited") diff --git a/resources_servers/terminal_bench_4/tests/test_environment.py b/resources_servers/terminal_bench_4/tests/test_environment.py new file mode 100644 index 0000000000..be626c741e --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_environment.py @@ -0,0 +1,292 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import json +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +import yaml + +from resources_servers.terminal_bench_4 import environment as module +from resources_servers.terminal_bench_4.compose_config import resolve_compose +from resources_servers.terminal_bench_4.environment import Environment, EnvironmentConfig, HealthcheckError +from resources_servers.terminal_bench_4.task import TaskSettings + + +def environment_config(**overrides): + return EnvironmentConfig.model_validate( + { + "cpu_enforcement_policy": "limit", + "memory_enforcement_policy": "limit", + "sandbox_provider": {"opensandbox": {"connection": {"domain": "example.invalid"}}}, + "sandbox_metadata": {}, + "sandbox_provider_options": {}, + "sandbox_env": {}, + "sandbox_env_by_task": {}, + "sandbox_request_gpu_type": True, + "sandbox_split_endpoints": False, + "compose_image_configs": None, + "sandbox_ttl_s": 21600, + "sandbox_ready_timeout_s": 900, + "default_exec_timeout_s": 1800, + "exec_shell": "bash -c", + "image_rewrites": [], + "workdir": None, + "efs_logs_host_path": None, + "efs_logs_init_image": "python:3.13-slim", + } + | overrides + ) + + +def make_environment(tmp_path, monkeypatch, *, compose=False, verifier=False, config=None, task_config=None): + raw = { + "environment": { + "docker_image": "public/agent", + "cpus": 2, + "memory_mb": 4096, + "storage_mb": 15000, + "env": {"TASK": "value"}, + }, + "agent": {"user": "task-user"}, + "verifier": {"environment": {"docker_image": "public/verifier", "cpus": 8, "gpus": 1, "gpu_types": ["H100"]}}, + } + for key, value in (task_config or {}).items(): + raw.setdefault(key, {}).update(value) + task = SimpleNamespace( + config=TaskSettings.model_validate(raw), name="terminal-bench/test", path=tmp_path / "package" + ) + (task.path / "environment").mkdir(parents=True, exist_ok=True) + (task.path / "tests").mkdir(exist_ok=True) + (task.path / "environment/Dockerfile").write_text("FROM public") + (task.path / "tests/Dockerfile").write_text("FROM public") + cfg = environment_config(**(config or {})) + if compose: + (task.path / "environment/docker-compose.yaml").write_text( + "services: {main: {image: public}, db: {image: db}}" + ) + cfg.compose_image_configs = tmp_path / "images.json" + cfg.compose_image_configs.write_text("{}") + monkeypatch.setattr( + module, + "resolve_compose", + lambda *a: {"services": {"main": {"image": "main", "shm_size": 64}, "db": {"image": "db"}}}, + ) + box = MagicMock() + box._handle = SimpleNamespace(sandbox_id="owned-box") + box.start = AsyncMock() + box.stop = AsyncMock() + box.exec = AsyncMock(return_value=SimpleNamespace(return_code=0, stdout="/app\n", stderr="")) + box.serialize = AsyncMock(return_value={"sandbox_id": "owned-box", "credentials": "must not be copied"}) + create = MagicMock(return_value=box) + monkeypatch.setattr(module, "AsyncSandbox", create) + group = SimpleNamespace(services={"main": box, "db": box}, start=AsyncMock(), stop=AsyncMock(), project="project") + compose_create = MagicMock(return_value=group) + monkeypatch.setattr(module, "AsyncSandboxCompose", compose_create) + env = Environment(task, cfg, "session", tmp_path / "result", verifier=verifier) + return env, box, create, compose_create + + +def test_specs_keep_resource_units_and_independent_verifier_pools(tmp_path, monkeypatch): + for pool in ["CPU", "GPU"]: + monkeypatch.setenv("OPENSANDBOX_DOMAIN_" + pool, pool + ".invalid") + monkeypatch.setenv("OPENSANDBOX_API_KEY_" + pool, "credential") + args = { + "sandbox_split_endpoints": True, + "sandbox_request_gpu_type": False, + "sandbox_provider_options": {"resource_requests": "limits"}, + "sandbox_env_by_task": {"test": {"OVERRIDE": "task"}}, + "sandbox_env": {"OVERRIDE": "global"}, + } + env, *_ = make_environment(tmp_path, monkeypatch, config=args) + verifier = Environment(env.task, env.config, "verify", tmp_path, verifier=True) + agent_spec, verifier_spec = env.build_spec(), verifier.build_spec() + assert env.pool == "cpu" and verifier.pool == "gpu" + assert agent_spec.resources.cpu == 2 and agent_spec.resources.memory_mib == 4096 + assert agent_spec.resources.disk_gib == 15 + assert verifier_spec.resources.gpu == 1 and verifier_spec.resources.gpu_type is None + assert agent_spec.env == {"TASK": "value", "OVERRIDE": "global"} + assert agent_spec.provider_options == {"resource_requests": "limits"} + assert "credential" not in str(agent_spec) + assert verifier.provider_config["opensandbox"]["connection"]["domain"] == "GPU.invalid" + + +async def test_single_start_descriptor_env_user_quiescence_cleanup(tmp_path, monkeypatch): + env, box, create, _ = make_environment(tmp_path, monkeypatch) + await env.start() + box.start.assert_awaited_once() + assert create.call_args.args[1].image == "public/agent" + descriptor = await env.main_connection() + assert descriptor == {"provider": "default", "sandbox_id": "owned-box", "workdir": "/app"} + assert box.exec.await_args.kwargs["user"] == "task-user" + await env.exec("echo test", env={"TASK": "changed"}, user="another", timeout_sec=12) + assert box.exec.await_args.kwargs["env"] == {"TASK": "changed"} + assert box.exec.await_args.kwargs["user"] == "another" + assert box.exec.await_args.kwargs["timeout_s"] == 12 + await env.quiesce_agent("session") + assert "/tmp/session.pids" in box.exec.await_args.args[0] + await env.stop() + await env.stop() + box.stop.assert_awaited_once() + assert env.closed and env.resources[0]["sandbox_id"] == "owned-box" + + +async def test_compose_specs_startup_metadata_sidecar_operations(tmp_path, monkeypatch): + env, box, _, create = make_environment(tmp_path, monkeypatch, compose=True) + await env.start() + kwargs = create.call_args.kwargs + assert kwargs["service_specs"]["main"].resources.cpu == 2 + assert kwargs["service_specs"]["db"].resources.cpu is None + document = yaml.safe_load(create.call_args.args[1].read_text()) + assert document["services"]["main"]["labels"] == {"nemo.nvidia.com/shm": "64"} + await env.exec("echo sidecar", service="db") + assert box.exec.await_args.args[0].startswith("sh -c ") + assert box.exec.await_args.kwargs["env"] is None + await env.stop_main() + await env.stop() + assert env.closed and env.resources[-1]["compose_project"] == "project" + + +@pytest.mark.parametrize( + "task_name,service,user,extension", + [ + ( + "medical-claims-processing", + "playwright-mcp", + "pwuser", + {"hosts": [], "resolve_environment": ["BROWSER_URL"]}, + ), + ("payments-pipeline-fix", "kafka", "appuser", {"hosts": []}), + ], +) +@pytest.mark.parametrize("scope", ["agent", "verifier", "other-task"]) +async def test_nonroot_compose_adaptations_are_scoped_to_agent_tasks( + tmp_path, monkeypatch, task_name, service, user, extension, scope +): + env, _, _, create = make_environment(tmp_path, monkeypatch, compose=True) + env.task.name = "terminal-bench/" + (task_name if scope != "other-task" else "unrelated") + if scope == "verifier": + env.log_role = "verifier" + env.environment_dir = env.task.path / "tests" + document = { + "services": { + "main": {"depends_on": {service: {"condition": "service_healthy"}}}, + service: { + "image": "nonroot", + "environment": {"BROWSER_URL": "http://workspace:18073"}, + "healthcheck": {"test": ["CMD", "true"]}, + }, + "workspace": {"image": "public/agent"}, + } + } + source = env.environment_dir / "docker-compose.yaml" + source.write_text(yaml.safe_dump(document)) + original = source.read_bytes() + images = { + image: { + "image": image, + "os": "linux", + "architecture": "amd64", + "config": {"User": image_user, "Cmd": ["sleep", "infinity"]}, + } + for image, image_user in [("public/agent", "root"), ("nonroot", user)] + } + env.config.compose_image_configs.write_text(json.dumps(images)) + monkeypatch.setattr(module, "resolve_compose", resolve_compose) + expected = resolve_compose(deepcopy(document), "public/agent", images) + if scope == "agent": + expected["services"][service]["x-sandbox"] = extension + expected["services"][service].pop("user") + + await env.start() + + generated = yaml.safe_load(create.call_args.args[1].read_text()) + assert generated == expected + assert generated["services"][service].get("user") == (None if scope == "agent" else user) + assert source.read_bytes() == original + + +async def test_environment_upload_without_build_spec(tmp_path, monkeypatch): + env, box, _, _ = make_environment(tmp_path, monkeypatch) + (env.environment_dir / "Dockerfile").unlink() + (env.environment_dir / "asset.txt").write_text("asset") + from resources_servers.terminal_bench_4 import transfers + + upload = AsyncMock() + monkeypatch.setattr(transfers, "upload_dir", upload) + await env.start() + upload.assert_awaited_once_with(box, env.environment_dir, "/app") + + +@pytest.mark.parametrize("failure", ["logs", "descriptor_id", "descriptor_pwd", "quiesce", "delete", "unavailable"]) +async def test_failures_are_visible_and_preserve_cleanup_identities(tmp_path, monkeypatch, failure): + env, box, _, _ = make_environment(tmp_path, monkeypatch) + if failure == "unavailable": + with pytest.raises(RuntimeError): + env.sandbox() + with pytest.raises(ValueError): + env.sandbox("missing") + return + if failure == "logs": + box.exec.return_value.return_code = 1 + if failure == "logs": + with pytest.raises(RuntimeError, match="log directories"): + await env.start() + else: + await env.start() + if failure == "descriptor_id": + box.serialize.return_value = {} + with pytest.raises(ValueError): + await env.main_connection() + if failure == "descriptor_pwd": + box.exec.return_value.return_code = 1 + with pytest.raises(RuntimeError): + await env.main_connection() + if failure == "quiesce": + box.exec.return_value.return_code = 1 + with pytest.raises(RuntimeError): + await env.quiesce_agent("session") + if failure == "delete": + box.stop.side_effect = RuntimeError("delete failed") + with pytest.raises(RuntimeError): + await env.stop() + assert env.cleanup_errors[0]["resources"][0]["sandbox_id"] == "owned-box" + else: + await env.stop() + + +async def test_readiness_success_and_failure(tmp_path, monkeypatch): + env, box, _, _ = make_environment( + tmp_path, + monkeypatch, + task_config={"environment": {"healthcheck": {"command": "ready", "retries": 2, "interval_sec": 0}}}, + ) + await env.start() + await env.healthcheck() + box.exec.return_value.return_code = 1 + with pytest.raises(HealthcheckError): + await env.healthcheck() + env.settings.healthcheck.start_period_sec = 0.01 + env.settings.healthcheck.start_interval_sec = 0.01 + with pytest.raises(HealthcheckError): + await env.healthcheck() + + +def test_offline_policy_and_unsupported_config(tmp_path, monkeypatch): + env, *_ = make_environment( + tmp_path, + monkeypatch, + verifier=True, + task_config={"verifier": {"environment": {"docker_image": "offline", "allow_internet": False}}}, + ) + assert env.build_spec().provider_options["network_policy"] == {"defaultAction": "deny", "egress": []} + with pytest.raises(ValueError, match="Split endpoints"): + Environment( + env.task, environment_config(sandbox_provider={"local": {}}, sandbox_split_endpoints=True), "id", tmp_path + ) + with pytest.raises(ValueError, match="Missing environment"): + make_environment(tmp_path, monkeypatch, config={"sandbox_split_endpoints": True}) + with pytest.raises(ValueError, match="Offline"): + make_environment(tmp_path, monkeypatch, compose=True, task_config={"environment": {"allow_internet": False}}) diff --git a/resources_servers/terminal_bench_4/tests/test_handoff.py b/resources_servers/terminal_bench_4/tests/test_handoff.py new file mode 100644 index 0000000000..7df61882f8 --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_handoff.py @@ -0,0 +1,568 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from nemo_gym.openai_utils import NeMoGymResponseCreateParamsNonStreaming +from nemo_gym.sandbox import AsyncSandbox, SandboxHandle +from nemo_gym.sandbox.agent import empty_response +from nemo_gym.sandbox.handoff import AgentTermination, SandboxedVerifyRequest, SessionRequest +from nemo_gym.server_utils import SESSION_ID_KEY, ServerClient +from resources_servers.terminal_bench_4 import lifecycle +from resources_servers.terminal_bench_4.app import ( + TerminalBench4Config, + TerminalBench4ResourcesServer, + TerminalBench4SeedRequest, +) +from resources_servers.terminal_bench_4.task import TaskSettings +from resources_servers.terminal_bench_4.tests.test_environment import environment_config + + +@pytest.fixture +async def fixture(tmp_path, monkeypatch): + server = TerminalBench4ResourcesServer( + config=TerminalBench4Config( + host="localhost", + port=1, + name="tb4", + entrypoint="app.py", + environment=environment_config(sandbox_provider={"local": {}}), + artifacts_dir=tmp_path, + agent_max_timeout_sec=2, + shutdown_timeout_sec=0.01, + ), + server_client=MagicMock(spec=ServerClient), + ) + pin = next(iter(server._tasks.values())) + body = TerminalBench4SeedRequest( + task_name="terminal-bench/" + pin["name"], + task_ref=pin["ref"], + dataset_ref=server._manifest["ref"], + rollout_id="rollout", + ) + request = SimpleNamespace(session={SESSION_ID_KEY: "owner"}) + config = TaskSettings.model_validate( + { + "environment": {"docker_image": "agent"}, + "agent": {"timeout_sec": 28800, "user": "task-user"}, + "verifier": {"environment": {"docker_image": "verifier"}}, + } + ) + task = SimpleNamespace(config=config, instruction="Solve task") + server._loader.load = AsyncMock(return_value=task) + envs = [] + events = [] + + def create(task, config, session_id, directory, verifier=False): + name = "verifier" if verifier else "agent" + env = SimpleNamespace( + task=task, + session_id=session_id, + closed=False, + resources=[], + cleanup_errors=[], + shared_logs=None, + efs_logs_fallback=None, + build_spec=lambda: None, + resource_identities=lambda: [], + main=MagicMock(), + main_connection=AsyncMock(return_value={"provider": "gpu", "sandbox_id": "box", "workdir": "/task"}), + healthcheck=AsyncMock(), + quiesce_agent=AsyncMock(side_effect=lambda _: events.append("quiesce")), + ) + + async def start(): + events.append(name + "_start") + + async def stop(): + events.append(name + "_stop") + env.closed = True + + env.start, env.stop = AsyncMock(side_effect=start), AsyncMock(side_effect=stop) + envs.append(env) + return env + + monkeypatch.setattr(lifecycle, "Environment", create) + monkeypatch.setattr(lifecycle, "download_dir", AsyncMock()) + monkeypatch.setattr(lifecycle, "collect", AsyncMock(side_effect=lambda *a: events.append("collect"))) + monkeypatch.setattr(lifecycle, "restore", AsyncMock(side_effect=lambda *a: events.append("restore"))) + grade = AsyncMock(return_value={"rewards": {"reward": 0.75}}) + monkeypatch.setattr(lifecycle, "run_verifier", grade) + yield SimpleNamespace(server=server, request=request, body=body, envs=envs, grade=grade, events=events) + await lifecycle.shutdown(list(server._sessions.values()), 0.01) + + +@pytest.mark.parametrize("failure", [None, "start", "prepare", "cleanup", "unsupported"]) +async def test_efs_session_owns_helper_until_workloads_are_stopped(fixture, monkeypatch, failure): + f = fixture + f.server.config.environment.efs_logs_host_path = "/mnt/efs/data/shared" + logs = SimpleNamespace( + session_id="logs", + closed=False, + resources=[], + cleanup_errors=[], + resource_identities=lambda: [{"efs_subpath": "owned"}], + restored_archive="/logs/snapshot.tar.gz", + ) + + async def initialize(): + f.events.append("logs_start") + if failure == "start": + raise RuntimeError("logs start failed") + if failure == "unsupported": + raise RuntimeError("VOLUME::HOST_PATH_NOT_ALLOWED /mnt/efs/data/shared") + + async def prepare(): + assert f.envs[0].closed and not f.envs[1].closed + f.events.append("logs_prepare") + if failure == "prepare": + raise RuntimeError("snapshot unavailable; use host transfer") + + async def close(*, remove_data=True): + if failure != "unsupported": + assert all(env.closed for env in f.envs) + assert remove_data + f.events.append("logs_stop") + if failure == "cleanup": + raise RuntimeError("EFS cleanup failed") + logs.closed = True + + logs.start = AsyncMock(side_effect=initialize) + logs.prepare_verifier = AsyncMock(side_effect=prepare) + logs.stop = AsyncMock(side_effect=close) + monkeypatch.setattr(lifecycle, "SharedLogs", lambda env: logs) + if failure == "start": + with pytest.raises(HTTPException): + await seed(f) + f.grade.assert_not_awaited() + else: + session_id = await start(f) + response = await f.server.verify(f.request, verify_body(session_id)) + assert response.evaluation_completed and response.reward == 0.75 + assert response.infrastructure_error is None + if failure == "unsupported": + assert all(env.shared_logs is None for env in f.envs) + if failure == "cleanup": + session = f.server._sessions[session_id] + assert session.result["exception_info"]["exception_message"] == "EFS cleanup failed" + assert f.events.index("logs_start") < f.events.index("agent_start") + assert f.events.index("agent_stop") < f.events.index("logs_prepare") < f.events.index("verifier_start") + assert f.events[-1] == "logs_stop" + assert logs.stop.await_count == (2 if failure == "unsupported" else 1) + assert f.server._slots._value == f.server.config.max_concurrent_sessions + + +def verify_body(session_id, reason="completed"): + params = NeMoGymResponseCreateParamsNonStreaming(input=[]) + return SandboxedVerifyRequest( + session_id=session_id, + responses_create_params=params, + response=empty_response(params, "model"), + termination=AgentTermination(reason=reason), + ) + + +async def seed(f): + return await f.server.seed_session(f.request, f.body) + + +async def start(f): + result = await seed(f) + await f.server.start_session(f.request, SessionRequest(session_id=result.session_id)) + return result.session_id + + +def restart(f): + return TerminalBench4ResourcesServer(config=f.server.config, server_client=MagicMock(spec=ServerClient)) + + +async def test_pins_duplicates_handoff_verification_and_restart(fixture): + f = fixture + first, duplicate = await asyncio.gather(seed(f), seed(f)) + assert first == duplicate + assert first.sandbox.provider == "gpu" and first.sandbox.workdir == "/task" + assert first.user == "task-user" and first.agent_timeout_sec == 2 and first.setup_timeout_sec == 360 + f.server._loader.load.assert_awaited_once_with(f.body.task_name, f.body.task_ref) + f.envs[0].main_connection.assert_awaited_once() + directory = f.server._sessions[first.session_id].directory + assert all((directory / name).is_dir() for name in ("agent", "verifier", "artifacts/logs/artifacts")) + budgets = [await f.server.start_session(f.request, SessionRequest(session_id=first.session_id)) for _ in range(2)] + assert 0 < budgets[1]["agent_timeout_sec"] <= budgets[0]["agent_timeout_sec"] <= 2 + verified, retry = await asyncio.gather( + *(f.server.verify(f.request, verify_body(first.session_id)) for _ in range(2)) + ) + assert verified == retry and verified.reward == 0.75 and verified.evaluation_completed + assert verified.infrastructure_error is None + assert "provenance" not in verified.model_dump() + assert f.events == [ + "agent_start", + "quiesce", + "collect", + "agent_stop", + "verifier_start", + "restore", + "verifier_stop", + ] + f.grade.assert_awaited_once() + assert await restart(f).verify(f.request, verify_body(first.session_id, "timeout")) == verified + with pytest.raises(HTTPException): + await restart(f).seed_session(f.request, f.body) + assert f.server._slots._value == f.server.config.max_concurrent_sessions + + +@pytest.mark.parametrize( + "field,value", [("task_name", "../../secret"), ("task_ref", "latest"), ("dataset_ref", "wrong")] +) +async def test_untrusted_pins_rejected_before_allocation(fixture, field, value): + f = fixture + with pytest.raises(HTTPException) as exc: + await f.server.seed_session(f.request, f.body.model_copy(update={field: value})) + assert exc.value.status_code == 422 + f.server._loader.load.assert_not_awaited() + + +async def test_cookie_isolation_and_worker_conflict(fixture): + f = fixture + f.body = f.body.model_copy(update={"client_session_id": "stable", "execution_id": "worker1"}) + first = await seed(f) + new_request = SimpleNamespace(session={SESSION_ID_KEY: "another-cookie"}) + assert await f.server.seed_session(new_request, f.body) == first + with pytest.raises(HTTPException) as exc: + await f.server.seed_session(new_request, f.body.model_copy(update={"execution_id": "worker2"})) + assert exc.value.status_code == 409 + with pytest.raises(HTTPException) as exc: + await f.server.verify(SimpleNamespace(session={SESSION_ID_KEY: "stranger"}), verify_body(first.session_id)) + assert exc.value.status_code == 404 + f.server._loader.load.assert_awaited_once() + + +async def test_setup_cancel_does_not_grade_and_premature_verify_rejected(fixture): + f = fixture + session_id = (await seed(f)).session_id + with pytest.raises(HTTPException) as exc: + await f.server.verify(f.request, verify_body(session_id)) + assert exc.value.status_code == 409 + for _ in range(2): + assert await f.server.cancel_session(f.request, SessionRequest(session_id=session_id)) == { + "session_id": session_id, + "phase": "closed", + } + f.grade.assert_not_awaited() + assert f.envs[0].closed + result = await restart(f).verify(f.request, verify_body(session_id)) + assert not result.evaluation_completed and result.termination.reason == "cancelled" + + +async def test_start_rejected_after_setup_cancellation_wins(fixture): + f = fixture + session_id = (await seed(f)).session_id + session = f.server._sessions[session_id] + # Request cancellation without yielding to the cleanup task: start must + # honor the accepted cancellation even while the phase is still ready. + finish = await lifecycle.request_finish(session, AgentTermination(reason="cancelled")) + with pytest.raises(HTTPException) as exc: + await lifecycle.start_session(session) + assert exc.value.status_code == 409 + await finish + assert session.termination.reason == "cancelled" + assert "agent_execution" not in session.result + f.grade.assert_not_awaited() + + +@pytest.mark.parametrize("phase", ["seed", "verify"]) +async def test_http_disconnect_does_not_cancel_resource_work(fixture, phase): + f = fixture + gate = asyncio.Event() + if phase == "seed": + task = f.server._loader.load.return_value + + async def load(*args): + await gate.wait() + return task + + f.server._loader.load.side_effect = load + call = asyncio.create_task(seed(f)) + else: + session_id = await start(f) + + async def grade(*args): + await gate.wait() + return {"rewards": {"reward": 1}} + + f.grade.side_effect = grade + call = asyncio.create_task(f.server.verify(f.request, verify_body(session_id))) + await asyncio.sleep(0) + await asyncio.sleep(0) + call.cancel() + await asyncio.gather(call, return_exceptions=True) + gate.set() + if phase == "seed": + assert (await seed(f)).instruction == "Solve task" + else: + assert (await f.server.verify(f.request, verify_body(session_id))).reward == 1 + f.grade.assert_awaited_once() + + +@pytest.mark.parametrize("reason", ["completed", "cancelled", "timeout", "nonzero_exit", "infrastructure_error"]) +@pytest.mark.parametrize("reward", [0, 1, None]) +async def test_official_grades_and_agent_failure(fixture, reason, reward): + f = fixture + f.grade.return_value = {"rewards": {} if reward is None else {"reward": reward}} + session_id = await start(f) + result = await f.server.verify(f.request, verify_body(session_id, reason)) + assert result.reward == (reward or 0) + assert result.evaluation_completed == (reward is not None) + assert result.termination.reason == reason + assert bool(result.infrastructure_error) == (reward is None or reason == "infrastructure_error") + + +async def test_deadline_grades_without_worker_and_late_first_verify_replays(fixture): + f = fixture + f.server.config.agent_max_timeout_sec = 0.01 + session_id = await start(f) + await asyncio.sleep(0.04) + session = f.server._sessions[session_id] + await session.finalization + assert session.phase == "closed" and session.verify_body is None + f.grade.assert_awaited_once() + saved = json.loads(f.server._state_path(session.identity).read_text()) + body = verify_body(session_id) + body.termination.artifacts = ["worker/trajectory.json"] + response = await restart(f).verify(f.request, body) + assert response.termination.reason == "timeout" + assert response.termination.artifacts == body.termination.artifacts + assert response.evaluation_completed + assert await restart(f).verify(f.request, verify_body(session_id, "cancelled")) == response + replayed = json.loads(f.server._state_path(session.identity).read_text()) + for field in ("resources", "deadlines", "diagnostics"): + assert replayed[field] == saved[field] + + +async def test_expired_deadline_wins_even_when_watchdog_delayed(fixture): + f = fixture + session_id = await start(f) + session = f.server._sessions[session_id] + session.watchdog.cancel() + session.deadline = 0 + result = await f.server.verify(f.request, verify_body(session_id)) + assert result.termination.reason == "timeout" + f.grade.assert_awaited_once() + + +@pytest.mark.parametrize("first", ["verify", "cancel"]) +async def test_finish_race_first_winner_and_exact_response(fixture, first): + f = fixture + session_id = await start(f) + calls = [ + f.server.verify(f.request, verify_body(session_id)), + f.server.cancel_session(f.request, SessionRequest(session_id=session_id)), + ] + if first == "cancel": + calls.reverse() + await asyncio.gather(*calls) + result = await f.server.verify(f.request, verify_body(session_id, "infrastructure_error")) + assert result.termination.reason == ("completed" if first == "verify" else "cancelled") + f.grade.assert_awaited_once() + + +async def test_setup_budget_excludes_queue_and_provisioning_includes_descriptor(fixture, monkeypatch): + f = fixture + monkeypatch.setattr(lifecycle, "SETUP_TIMEOUT_SEC", 0.03) + await f.server._slots.acquire() + f.server._slots = asyncio.Semaphore(0) + call = asyncio.create_task(seed(f)) + await asyncio.sleep(0.04) + f.server._slots.release() + result = await call + session = f.server._sessions[result.session_id] + assert session.phase == "ready" + await asyncio.sleep(0.06) + await session.finalization + assert session.phase == "closed" + assert session.result["exception_info"]["exception_type"] == "AgentSetupTimeoutError" + f.grade.assert_not_awaited() + + +@pytest.mark.parametrize( + "failure", ["load", "start", "healthcheck", "descriptor", "quiesce", "collect", "verifier", "cleanup"] +) +async def test_failures_release_slots_and_attempt_remaining_cleanup(fixture, monkeypatch, failure): + f = fixture + if failure == "load": + f.server._loader.load.side_effect = RuntimeError("load failed") + else: + original = lifecycle.Environment + + def create(*args, **kwargs): + env = original(*args, **kwargs) + method = { + "start": "start", + "healthcheck": "healthcheck", + "descriptor": "main_connection", + "quiesce": "quiesce_agent", + "cleanup": "stop", + }.get(failure) + if method and not kwargs.get("verifier"): + getattr(env, method).side_effect = RuntimeError(failure + " failed") + return env + + monkeypatch.setattr(lifecycle, "Environment", create) + if failure in {"load", "start", "healthcheck", "descriptor"}: + with pytest.raises(HTTPException): + await seed(f) + else: + session_id = await start(f) + if failure == "collect": + monkeypatch.setattr(lifecycle, "collect", AsyncMock(side_effect=RuntimeError("collection failed"))) + if failure == "verifier": + f.grade.side_effect = RuntimeError("verification failed") + result = await f.server.verify(f.request, verify_body(session_id)) + assert result.evaluation_completed == (failure == "cleanup") + session = next(iter(f.server._sessions.values())) + assert session.phase == "closed" and not session.owns_slot + if f.envs: + f.envs[-1].stop.assert_awaited() + if failure == "quiesce": + f.grade.assert_not_awaited() + + +async def test_shutdown_and_interrupted_restart(fixture): + f = fixture + app = f.server.setup_webserver() + async with app.router.lifespan_context(app): + result = await seed(f) + with pytest.raises(HTTPException) as exc: + restart(f)._session(f.request, result.session_id) + assert exc.value.status_code == 409 + assert f.envs[0].closed + with pytest.raises(HTTPException) as exc: + await seed(f) + assert exc.value.status_code == 503 + + +async def test_legacy_closed_result_without_response(fixture): + f = fixture + session_id = await start(f) + result = await f.server.verify(f.request, verify_body(session_id)) + session = f.server._sessions[session_id] + path = f.server._state_path(session.identity) + state = json.loads(path.read_text()) + state.pop("record_version") + state.pop("verify_body") + state["verified_response"] = None + state["result"].pop("runtime") + state["result"].pop("runtime_version") + path.write_text(json.dumps(state)) + replay = await restart(f).verify(f.request, verify_body(session_id)) + assert replay.reward == result.reward and replay.evaluation_completed + assert "provenance" not in replay.model_dump() + + +@pytest.mark.parametrize("owned,operation", [(False, "release"), (False, "stop"), (True, "stop")]) +async def test_borrowed_handles_do_not_destroy_owned_collection(owned, operation): + provider = MagicMock(close=AsyncMock(), aclose=AsyncMock()) + sandbox = AsyncSandbox(provider, owns_sandbox=owned) + sandbox._handle = SandboxHandle(sandbox_id="box", provider_name="test", raw=None) + sandbox._stopped = False + await getattr(sandbox, operation)() + await sandbox.stop() + assert provider.close.await_count == int(owned) + provider.aclose.assert_awaited_once() + + +async def test_descriptor_timeout_is_owned_and_cleans_partial_preparation(fixture, monkeypatch): + f = fixture + monkeypatch.setattr(lifecycle, "SETUP_TIMEOUT_SEC", 0.01) + original = lifecycle.Environment + + def create(*args, **kwargs): + env = original(*args, **kwargs) + + async def blocked(): + await asyncio.Event().wait() + + env.main_connection.side_effect = blocked + return env + + monkeypatch.setattr(lifecycle, "Environment", create) + with pytest.raises(HTTPException): + await seed(f) + session = next(iter(f.server._sessions.values())) + await session.finalization + assert f.envs[0].closed and not session.owns_slot + assert session.result["exception_info"]["exception_type"] == "AgentSetupTimeoutError" + assert session.result["agent_setup"]["finished_at"] + f.grade.assert_not_awaited() + + +async def test_delayed_start_cannot_extend_setup_or_execution(fixture): + f = fixture + session_id = (await seed(f)).session_id + session = f.server._sessions[session_id] + session.watchdog.cancel() + session.setup_deadline = 0 + with pytest.raises(HTTPException): + await f.server.start_session(f.request, SessionRequest(session_id=session_id)) + assert session.phase == "closed" + f.body = f.body.model_copy(update={"rollout_id": "second"}) + session_id = await start(f) + session = f.server._sessions[session_id] + session.watchdog.cancel() + session.deadline = 0 + with pytest.raises(HTTPException): + await f.server.start_session(f.request, SessionRequest(session_id=session_id)) + assert session.termination.reason == "timeout" + f.grade.assert_awaited_once() + + +async def test_shutdown_interrupts_verification_and_preserves_cleanup(fixture): + f = fixture + session_id = await start(f) + entered = asyncio.Event() + + async def blocked(*args): + entered.set() + await asyncio.Event().wait() + + f.grade.side_effect = blocked + verify = asyncio.create_task(f.server.verify(f.request, verify_body(session_id))) + await entered.wait() + await lifecycle.shutdown(list(f.server._sessions.values()), 0.01) + result = await verify + assert not result.evaluation_completed and result.infrastructure_error == "CancelledError" + assert all(env.closed for env in f.envs) + assert not f.server._sessions[session_id].owns_slot + + +async def test_build_timeout_and_missing_agent_logs_are_diagnostic(fixture, monkeypatch): + f = fixture + f.server._loader.load.return_value.config.environment.build_timeout_sec = 0.01 + original = lifecycle.Environment + + def create(*args, **kwargs): + env = original(*args, **kwargs) + + async def blocked(): + await asyncio.Event().wait() + + env.start.side_effect = blocked + return env + + monkeypatch.setattr(lifecycle, "Environment", create) + with pytest.raises(HTTPException): + await seed(f) + session = next(iter(f.server._sessions.values())) + assert session.result["exception_info"]["exception_type"] == "EnvironmentStartTimeoutError" + monkeypatch.setattr(lifecycle, "Environment", original) + monkeypatch.setattr(lifecycle, "download_dir", AsyncMock(side_effect=RuntimeError("logs unavailable"))) + f.body = f.body.model_copy(update={"rollout_id": "second"}) + session_id = await start(f) + response = await f.server.verify(f.request, verify_body(session_id)) + assert response.evaluation_completed + assert any(d.get("operation") == "agent_logs" for d in f.server._sessions[session_id].diagnostics) diff --git a/resources_servers/terminal_bench_4/tests/test_shared_logs.py b/resources_servers/terminal_bench_4/tests/test_shared_logs.py new file mode 100644 index 0000000000..5f1859402d --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_shared_logs.py @@ -0,0 +1,312 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import os +import shlex +import shutil +import sys +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nemo_gym.sandbox import SandboxSpec +from resources_servers.terminal_bench_4 import shared_logs as module +from resources_servers.terminal_bench_4.collection import collect +from resources_servers.terminal_bench_4.shared_logs import SharedLogs +from resources_servers.terminal_bench_4.task import TaskSettings +from resources_servers.terminal_bench_4.tests.test_collection_verifier import environment +from resources_servers.terminal_bench_4.tests.test_environment import environment_config, make_environment +from resources_servers.terminal_bench_4.verifier import restore, run_verifier + + +class LocalHelper: + """Execute the production helper scripts against a temporary EFS tree.""" + + def __init__(self): + self.start = AsyncMock() + self.stop = AsyncMock() + self._handle = SimpleNamespace(sandbox_id="helper") + + async def exec(self, command, **kwargs): + args = shlex.split(command) + assert args[0] == "python3" + process = await asyncio.create_subprocess_exec( + sys.executable, *args[1:], stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await process.communicate() + return SimpleNamespace(return_code=process.returncode, stdout=stdout.decode(), stderr=stderr.decode()) + + +def shared(tmp_path, monkeypatch): + helper = LocalHelper() + constructor = MagicMock(return_value=helper) + monkeypatch.setattr(module, "AsyncSandbox", constructor) + mount = tmp_path / "efs" + mount.mkdir(exist_ok=True) + monkeypatch.setattr(SharedLogs, "mount", str(mount)) + config = environment_config( + sandbox_provider={"opensandbox": {"connection": {"domain": "example.invalid"}}}, + efs_logs_host_path="/mnt/efs/data/shared", + ) + env = SimpleNamespace( + config=config, + provider_config=config.sandbox_provider, + pool="default", + session_id="episode", + build_spec=lambda: SandboxSpec(image="task", metadata={"tb4-session": "episode", "run": "test"}), + ) + return SharedLogs(env), helper, constructor + + +def mount_role(env, logs, role): + shutil.rmtree(env.main.path("/logs")) + env.main.path("/logs").symlink_to(Path(logs.root) / role, target_is_directory=True) + for directory in ("agent", "verifier", "artifacts"): + env.main.path("/logs/" + directory).mkdir() + env.shared_logs = logs + + +async def test_efs_snapshot_matches_host_restore_and_survives_agent_deletion(tmp_path, monkeypatch): + logs, helper, _ = shared(tmp_path, monkeypatch) + await logs.start() + cfg = TaskSettings.model_validate( + { + "environment": {"docker_image": "agent"}, + "verifier": {"environment": {"docker_image": "verifier"}}, + "artifacts": [{"source": "/logs/artifacts", "exclude": ["*.secret"]}, {"source": "/app"}], + } + ) + agent, box, _ = environment(tmp_path / "agent", cfg) + verifier, target, _ = environment(tmp_path / "verifier", cfg) + mount_role(agent, logs, "agent") + mount_role(verifier, logs, "verifier") + box.path("/logs/agent/trajectory.json").write_text("private agent logs") + box.path("/logs/verifier/reward.txt").write_text("999") + box.path("/logs/undeclared.txt").write_text("not an artifact") + box.path("/logs/artifacts/report.txt").write_bytes(b"original\x00bytes") + box.path("/logs/artifacts/report.txt").chmod(0o751) + box.path("/logs/artifacts/link").symlink_to("report.txt") + box.path("/logs/artifacts/empty").mkdir() + box.path("/logs/artifacts/omit.secret").write_text("excluded") + box.path("/app/source.txt").write_text("normal artifact") + artifacts = tmp_path / "artifacts" + diagnostics = [] + await collect(agent, artifacts, diagnostics) + assert logs.archive_digest + assert not diagnostics + # Neither subsequent writes nor deleting the workload can affect the + # already collected snapshot. EFS persists independently of the sandbox. + box.path("/logs/artifacts/report.txt").write_text("late mutation") + shutil.rmtree(box.root) + await logs.prepare_verifier() + assert logs.restored_archive + target.upload = AsyncMock(wraps=target.upload) + await restore(verifier, artifacts) + assert target.path("/logs/artifacts/report.txt").read_bytes() == b"original\x00bytes" + assert target.path("/logs/artifacts/report.txt").stat().st_mode & 0o777 == 0o751 + assert target.path("/logs/artifacts/link").is_symlink() + assert target.path("/logs/artifacts/link").read_bytes() == b"original\x00bytes" + assert target.path("/logs/artifacts/empty").is_dir() + assert not target.path("/logs/artifacts/omit.secret").exists() + assert not target.path("/logs/undeclared.txt").exists() + assert not list(target.path("/logs/agent").iterdir()) + assert not list(target.path("/logs/verifier").iterdir()) + assert not target.path(logs.restored_archive).exists() + assert target.path("/app/source.txt").read_text() == "normal artifact" + # Only /app was uploaded; /logs/artifacts came from the EFS snapshot. + assert target.upload.await_count == 1 + target.path("/tests/test.sh").write_text(f"#!/bin/sh\necho 1 > {target.path('/logs/verifier/reward.txt')}\n") + assert await run_verifier(verifier, tmp_path / "result", diagnostics) == {"rewards": {"reward": 1}} + await logs.stop() + await logs.stop() + assert not Path(logs.root).exists() + assert logs.closed + helper.stop.assert_awaited_once() + assert (artifacts / "logs/artifacts/report.txt").read_bytes() == b"original\x00bytes" + + +async def test_archive_tampering_uses_collected_host_fallback(tmp_path, monkeypatch): + logs, _, _ = shared(tmp_path, monkeypatch) + await logs.start() + cfg = TaskSettings.model_validate( + {"environment": {"docker_image": "a"}, "verifier": {"environment_mode": "separate"}} + ) + agent, box, _ = environment(tmp_path / "agent", cfg) + verifier, target, _ = environment(tmp_path / "verifier", cfg) + mount_role(agent, logs, "agent") + mount_role(verifier, logs, "verifier") + box.path("/logs/artifacts/result.txt").write_text("snapshot") + await collect(agent, tmp_path / "artifacts", []) + box.path("/logs/" + logs.archive_name).write_text("changed after collection") + await logs.prepare_verifier() + assert logs.restored_archive is None + await restore(verifier, tmp_path / "artifacts") + assert target.path("/logs/artifacts/result.txt").read_text() == "snapshot" + await logs.stop() + + +@pytest.mark.parametrize( + "other", + [ + {"source": "/logs"}, + {"source": "/logs/artifacts/nested"}, + {"source": "/logs/artifacts", "service": "db"}, + {"source": "/app", "destination": "logs/artifacts"}, + ], +) +def test_overlapping_artifacts_keep_ordered_host_restore(tmp_path, monkeypatch, other): + logs, _, _ = shared(tmp_path, monkeypatch) + cfg = TaskSettings.model_validate( + {"environment": {"docker_image": "a"}, "verifier": {"environment_mode": "separate"}, "artifacts": [other]} + ) + artifacts = cfg.collected_artifacts + assert logs.collection_archive(artifacts[0], artifacts) is None + + +async def test_roles_and_episodes_are_isolated_and_writable(tmp_path, monkeypatch): + first, _, create = shared(tmp_path, monkeypatch) + second, _, _ = shared(tmp_path, monkeypatch) + await first.start() + await second.start() + assert first.volume("agent")["subPath"] != first.volume("verifier")["subPath"] + assert first.volume("agent")["subPath"] != second.volume("agent")["subPath"] + assert first.volume("agent")["mountPath"] == "/logs" + for role in ("agent", "verifier"): + assert (Path(first.root) / role).stat().st_mode & 0o777 == 0o777 + spec = create.call_args.args[1] + assert spec.resources.cpu == 1 and not spec.resources.gpu + assert spec.metadata["run"] == "test" + await first.stop() + assert Path(second.root).is_dir() + await second.stop() + + +async def test_partial_initialization_and_failed_delete_still_stop_helper(tmp_path, monkeypatch): + logs, helper, _ = shared(tmp_path, monkeypatch) + await logs.start() + # Treat a partly created role tree as a setup failure. + shutil.rmtree(Path(logs.root) / "verifier") + await logs.stop() + assert not Path(logs.root).exists() + helper.stop.assert_awaited_once() + failed, helper, _ = shared(tmp_path, monkeypatch) + await failed.start() + failed.python = AsyncMock(side_effect=RuntimeError("EFS unavailable")) + with pytest.raises(ExceptionGroup, match="cleanup failed"): + await failed.stop() + helper.stop.assert_awaited_once() + assert failed.cleanup_errors and not failed.closed + + +async def test_live_workload_mount_is_retained_on_failed_teardown(tmp_path, monkeypatch): + logs, helper, _ = shared(tmp_path, monkeypatch) + await logs.start() + await logs.stop(remove_data=False) + assert Path(logs.root).exists() + helper.stop.assert_awaited_once() + assert logs.resources[0]["efs_subpath"] == logs.relative + + +async def test_compose_mounts_logs_only_in_main(tmp_path, monkeypatch): + env, _, _, compose = make_environment(tmp_path, monkeypatch, compose=True) + logs, _, _ = shared(tmp_path, monkeypatch) + logs.initialize_role = AsyncMock() + env.shared_logs = logs + await env.start() + specs = compose.call_args.kwargs["service_specs"] + assert specs["main"].provider_options["volumes"] == [logs.volume("agent")] + assert "volumes" not in specs["db"].provider_options + + +async def test_log_owner_uses_image_identity_and_protects_parent(tmp_path, monkeypatch): + logs, _, _ = shared(tmp_path, monkeypatch) + await logs.start() + env = SimpleNamespace( + log_role="agent", + exec=AsyncMock(return_value=SimpleNamespace(return_code=0, stdout=f"{os.getuid()}\n{os.getgid()}\n")), + ) + await logs.initialize_role(env) + root = Path(logs.root) / "agent" + assert root.stat().st_uid == os.getuid() and root.stat().st_mode & 0o777 == 0o755 + assert "user" not in env.exec.await_args.kwargs + for result in (SimpleNamespace(return_code=0, stdout="invalid"), SimpleNamespace(return_code=1, stdout="0 0")): + env.exec.return_value = result + with pytest.raises(RuntimeError, match="task log owner"): + await logs.initialize_role(env) + await logs.prepare_verifier() # No snapshot: normal host restore. + await logs.stop() + + +@pytest.mark.parametrize("error", ["VOLUME::HOST_PATH_NOT_ALLOWED /mnt/efs/data/shared", "quota exceeded"]) +async def test_only_explicit_unsupported_efs_mount_uses_original_lifecycle(tmp_path, monkeypatch, error): + env, box, create, _ = make_environment(tmp_path, monkeypatch) + logs, _, _ = shared(tmp_path, monkeypatch) + logs.initialize_role = AsyncMock() + env.shared_logs = logs + box.start.side_effect = [RuntimeError(error), None] + if error.startswith("quota"): + with pytest.raises(RuntimeError, match="quota"): + await env.start() + assert env.shared_logs is logs and create.call_count == 1 + else: + await env.start() + assert env.shared_logs is None and env.efs_logs_fallback == error + assert create.call_count == 2 + assert "volumes" not in create.call_args.args[1].provider_options + box.stop.assert_awaited_once() + logs.initialize_role.assert_not_awaited() + + +def test_conflicting_log_mount_rejected_before_provisioning(tmp_path, monkeypatch): + env, _, create, _ = make_environment( + tmp_path, monkeypatch, config={"sandbox_provider_options": {"volumes": [{"mountPath": "/logs/"}]}} + ) + env.shared_logs, _, _ = shared(tmp_path, monkeypatch) + with pytest.raises(ValueError, match="conflict"): + env.build_spec() + create.assert_not_called() + + +async def test_cancelled_cleanup_continues_and_closes_helper(tmp_path, monkeypatch): + logs, helper, _ = shared(tmp_path, monkeypatch) + await logs.start() + cleanup_started, proceed = asyncio.Event(), asyncio.Event() + execute = logs.python + + async def delayed(*args): + cleanup_started.set() + await proceed.wait() + return await execute(*args) + + logs.python = delayed + pending = asyncio.create_task(logs.stop()) + await cleanup_started.wait() + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + assert Path(logs.root).exists() + proceed.set() + await logs.stop() + assert not Path(logs.root).exists() + helper.stop.assert_awaited_once() + + +def test_gpu_workload_uses_cpu_helper_and_independent_role_mounts(tmp_path, monkeypatch): + for pool in ("CPU", "GPU"): + monkeypatch.setenv(f"OPENSANDBOX_DOMAIN_{pool}", pool + ".invalid") + monkeypatch.setenv(f"OPENSANDBOX_API_KEY_{pool}", "credential") + env, *_ = make_environment(tmp_path, monkeypatch, config={"sandbox_split_endpoints": True}, verifier=True) + env.config.efs_logs_host_path = "/mnt/efs/data/shared" + create = MagicMock() + monkeypatch.setattr(module, "AsyncSandbox", create) + logs = SharedLogs(env) + cfg, spec = create.call_args.args + assert cfg["opensandbox"]["connection"]["domain"] == "CPU.invalid" + assert spec.metadata["nemo-gym.nvidia.com/resource-pool"] == "cpu" + assert spec.resources.gpu is None and not spec.env + assert env.provider_config["opensandbox"]["connection"]["domain"] == "GPU.invalid" + env.shared_logs = logs + assert env.build_spec().provider_options["volumes"] == [logs.volume("verifier")] diff --git a/resources_servers/terminal_bench_4/tests/test_task.py b/resources_servers/terminal_bench_4/tests/test_task.py new file mode 100644 index 0000000000..acdbfab1f7 --- /dev/null +++ b/resources_servers/terminal_bench_4/tests/test_task.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +import hashlib +import io +import shutil +import tarfile +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +from resources_servers.terminal_bench_4 import task as module +from resources_servers.terminal_bench_4.task import ( + Artifact, + PackageLoader, + Task, + TaskSettings, + content_hash, + resolve_env, +) + + +def package(path): + path.mkdir(parents=True) + (path / "environment").mkdir() + (path / "environment/Dockerfile").write_text("FROM public") + (path / "instruction.md").write_text("\n\nDo the task.\n") + (path / "task.toml").write_text( + '[task]\nname="terminal-bench/test"\n[environment]\ndocker_image="agent"\n[verifier.environment]\ndocker_image="verifier"\n' + ) + return path + + +def settings(): + return {"environment": {"docker_image": "agent"}, "verifier": {"environment": {"docker_image": "verifier"}}} + + +def test_content_hash_records_and_ignored_files(tmp_path): + path = package(tmp_path / "package") + records = [] + for name in ["environment/Dockerfile", "instruction.md", "task.toml"]: + digest = hashlib.sha256((path / name).read_bytes()).hexdigest() + records.append(f"{name}\0{digest}\n") + expected = hashlib.sha256("".join(records).encode()).hexdigest() + assert content_hash(path) == expected + (path / "environment/__pycache__").mkdir() + (path / "environment/__pycache__/code.pyc").write_bytes(b"ignored") + assert content_hash(path) == expected + (path / "environment/Dockerfile").write_text("FROM changed") + assert content_hash(path) != expected + + +async def test_warm_cache_revalidated_and_instruction_stripped(tmp_path): + path = package(tmp_path / "source") + ref = "sha256:" + content_hash(path) + dest = tmp_path / "terminal-bench/test" / ref[7:] + dest.parent.mkdir(parents=True) + path.rename(dest) + loader = PackageLoader(tmp_path) + loaded = await loader.load("terminal-bench/test", ref) + assert loaded.instruction == "Do the task.\n" + (dest / "environment/Dockerfile").write_text("FROM tampered") + with pytest.raises(ValueError, match="hash mismatch"): + await loader.load("terminal-bench/test", ref) + + +@pytest.mark.parametrize("name,ref", [("terminal-bench/test", "latest"), ("../test", "sha256:" + "a" * 64)]) +async def test_loader_rejects_unpinned_and_unsafe_names(tmp_path, name, ref): + with pytest.raises(ValueError): + await PackageLoader(tmp_path).load(name, ref) + + +class Response: + def __init__(self, *, data=None, blob=None): + self.data = data + self.blob = blob + self.content = self + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + def raise_for_status(self): + pass + + async def json(self): + return self.data + + async def iter_chunked(self, size): + yield self.blob + + +@pytest.mark.parametrize("failure", [None, "identity", "content", "layout"]) +async def test_cold_cache_atomic_download_and_shared_load(tmp_path, monkeypatch, failure): + source = package(tmp_path / "source") + ref = "sha256:" + content_hash(source) + if failure == "content": + (source / "instruction.md").write_text("changed") + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w:gz") as tar: + for p in source.rglob("*"): + if p.is_file(): + tar.add(p, arcname=("wrong/" if failure == "layout" else "") + str(p.relative_to(source))) + call = AsyncMock( + side_effect=[ + Response( + data={ + "content_hash": "0" * 64 if failure == "identity" else ref[7:], + "archive_path": "public/archive.tar.gz", + } + ), + Response(blob=stream.getvalue()), + ] + ) + monkeypatch.setattr(module, "request", call) + loader = PackageLoader(tmp_path / "cache") + if failure: + with pytest.raises((ValueError, FileNotFoundError)): + await loader.load("terminal-bench/test", ref) + assert not (loader.root / "terminal-bench/test" / ref[7:]).exists() + else: + a, b = await asyncio.gather(*(loader.load("terminal-bench/test", ref) for _ in range(2))) + assert a.path == b.path and a.instruction == "Do the task.\n" + assert call.await_count == 2 + assert call.await_args_list[0].kwargs["json"]["p_ref"] == ref + assert not list(loader.root.rglob(".tb4-download-*")) + + +@pytest.mark.parametrize("winner", [True, False]) +async def test_cache_promotion_race_or_filesystem_failure(tmp_path, monkeypatch, winner): + source = package(tmp_path / "source") + ref = "sha256:" + content_hash(source) + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w:gz") as tar: + for path in source.rglob("*"): + if path.is_file(): + tar.add(path, arcname=str(path.relative_to(source))) + monkeypatch.setattr( + module, + "request", + AsyncMock( + side_effect=[ + Response(data={"content_hash": ref[7:], "archive_path": "public/archive.tar.gz"}), + Response(blob=stream.getvalue()), + ] + ), + ) + + def promote(path, target): + if winner: + shutil.copytree(path, target) + raise FileExistsError("Another process promoted this pin") + raise PermissionError("Cache filesystem is read-only") + + monkeypatch.setattr(Path, "rename", promote) + loader = PackageLoader(tmp_path / "cache") + if winner: + loaded = await loader.load("terminal-bench/test", ref) + assert content_hash(loaded.path) == ref[7:] + else: + with pytest.raises(PermissionError): + await loader.load("terminal-bench/test", ref) + assert not list(loader.root.rglob(".tb4-download-*")) + + +@pytest.mark.parametrize( + "section,key,value", + [ + ("environment", "tpu", {}), + ("environment", "os", "windows"), + ("environment", "unknown_execution_field", 1), + ("environment", "network_mode", "allowlist"), + ("agent", "network_mode", "no-network"), + ("verifier", "environment_mode", "shared"), + ("verifier", "timeout_sec", float("inf")), + ], +) +def test_unsupported_execution_fields_fail_before_allocation(section, key, value): + data = settings() + data.setdefault(section, {})[key] = value + with pytest.raises(ValueError): + TaskSettings.model_validate(data) + + +def test_legacy_network_resource_fields_and_separate_inheritance(): + data = settings() + data["environment"].update(memory="1G", storage="1.5G", allow_internet=False) + data["verifier"] = {"environment_mode": "separate"} + cfg = TaskSettings.model_validate(data) + assert cfg.environment.memory_mb == 1024 and cfg.environment.storage_mb == 1536 + assert cfg.verifier_environment.network_mode == "no-network" + cfg.verifier_environment.env["local"] = "x" + assert cfg.environment.env == {} + data["environment"]["memory_mb"] = 2 + with pytest.raises(ValueError, match="Conflicting"): + TaskSettings.model_validate(data) + + +def test_mcp_skills_and_multistep_rejection(): + data = settings() + data["environment"].update( + mcp_servers=[{"name": "tools", "transport": "http", "url": "http://tools/mcp"}], skills_dir="/app/skills" + ) + cfg = TaskSettings.model_validate(data) + assert cfg.environment.mcp_servers[0].transport == "streamable-http" + assert cfg.environment.skills_dir == "/app/skills" + data["steps"] = [{"name": "step"}] + with pytest.raises(ValueError, match="single-step"): + TaskSettings.model_validate(data) + + +@pytest.mark.parametrize( + "value", + [ + {"source": "../bad"}, + {"source": "/ok", "destination": "/bad"}, + {"source": "/ok", "destination": "manifest.json"}, + {"source": "relative", "service": "db"}, + ], +) +def test_artifact_paths_are_contained(value): + with pytest.raises(ValueError): + Artifact.model_validate(value) + + +def test_destinations_and_conventional_directory_are_distinct(): + data = settings() + data["artifacts"] = [{"source": "/app/output", "destination": "saved", "exclude": ["*.tmp"]}, "/app/file"] + cfg = TaskSettings.model_validate(data) + assert cfg.collected_artifacts[0].source == "/logs/artifacts" + assert cfg.collected_artifacts[1].host_path == Path("saved") + assert cfg.collected_artifacts[1].source == "/app/output" + assert cfg.collected_artifacts[2].host_path == Path("app/file") + + +def test_env_templates_match_reference(monkeypatch): + monkeypatch.setenv("TB4_TEST_VALUE", "actual") + assert resolve_env( + {"a": "${TB4_TEST_VALUE}", "b": "${TB4_MISSING:-default}", "c": "prefix-${TB4_TEST_VALUE}"} + ) == { + "a": "actual", + "b": "default", + "c": "prefix-${TB4_TEST_VALUE}", + } + monkeypatch.delenv("TB4_MISSING", raising=False) + with pytest.raises(ValueError): + resolve_env({"a": "${TB4_MISSING}"}) + + +def test_links_cannot_escape_package(tmp_path): + path = package(tmp_path / "package") + (path / "environment/link").symlink_to(tmp_path / "outside") + (tmp_path / "outside").write_text("secret") + with pytest.raises(ValueError, match="escapes"): + Task.read(path, "terminal-bench/test", "sha256:" + "a" * 64) + with pytest.raises(ValueError, match="escapes"): + content_hash(path) diff --git a/resources_servers/terminal_bench_4/transfers.py b/resources_servers/terminal_bench_4/transfers.py new file mode 100644 index 0000000000..f10b99e2d4 --- /dev/null +++ b/resources_servers/terminal_bench_4/transfers.py @@ -0,0 +1,98 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""File and directory transfer through Gym sandbox operations.""" + +import hashlib +import shlex +import tarfile +import tempfile +from pathlib import Path, PurePosixPath +from uuid import uuid4 + + +async def upload_file(sandbox, source, target): + await sandbox.exec(f"mkdir -p {shlex.quote(str(PurePosixPath(target).parent))}", timeout_s=60) + await sandbox.upload(Path(source), target) + + +async def upload_dir(sandbox, source, target): + source = Path(source) + remote = f"/tmp/.nemo-gym-upload-{uuid4().hex}.tar.gz" + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "upload.tar.gz" + with tarfile.open(archive, "w:gz") as tar: + tar.add(source, arcname=".") + await sandbox.upload(archive, remote) + result = await sandbox.exec( + f"mkdir -p {shlex.quote(target)} && tar -xzf {remote} -C {shlex.quote(target)}; " + f"status=$?; rm -f {remote}; exit $status", + timeout_s=600, + ) + if result.return_code: + for path in sorted(source.rglob("*")): + destination = str(PurePosixPath(target) / path.relative_to(source).as_posix()) + if path.is_dir(): + await sandbox.exec(f"mkdir -p {shlex.quote(destination)}", timeout_s=60) + elif path.is_file(): + await upload_file(sandbox, path, destination) + + +async def download_file(sandbox, source, target): + target = Path(target) + target.parent.mkdir(parents=True, exist_ok=True) + await sandbox.download(source, target) + + +async def download_dir(sandbox, source, target, *, exclude=None, exec_command=None, shared_archive=None): + target = Path(target) + target.mkdir(parents=True, exist_ok=True) + remote = shared_archive or f"/tmp/.nemo-gym-download-{uuid4().hex}.tar.gz" + retained = False + flags = " ".join(f"--exclude={shlex.quote(pattern)}" for pattern in (exclude or [])) + command = f"tar -czf {remote} {flags} -C {shlex.quote(source)} ." + # The reference uses role shell/root for exclusions; plain directory + # transfers use the provider's default execution user and shell. + if exclude and exec_command: + result = await exec_command(command, timeout_sec=600, user="root") + else: + result = await sandbox.exec(command, timeout_s=600) + try: + if result.return_code: + if exclude: + raise RuntimeError(f"Failed to archive {source}: {result.stderr}") + listing = await sandbox.exec(f"find {shlex.quote(source)} -type f", timeout_s=120) + if listing.return_code: + raise RuntimeError(f"Failed to list {source}: {listing.stderr}") + for line in (listing.stdout or "").splitlines(): + if line.strip(): + relative = PurePosixPath(line.strip()).relative_to(PurePosixPath(source)) + await download_file(sandbox, line.strip(), target / relative) + return + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "download.tar.gz" + await sandbox.download(remote, archive) + with tarfile.open(archive, "r:gz") as tar: + tar.extractall(target, filter="data") + if shared_archive: + with archive.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + retained = True + return digest + finally: + if not retained: + await sandbox.exec(f"rm -f {remote}", timeout_s=60) + + +async def prepare_directory(environment, path, *, empty=False): + quoted = shlex.quote(path) + commands = [] + if empty: + commands.append(f"if [ -L {quoted} ] || {{ [ -e {quoted} ] && [ ! -d {quoted} ]; }}; then rm -rf {quoted}; fi") + commands.append(f"mkdir -p {quoted}") + if empty: + commands.append(f"find {quoted} -mindepth 1 -maxdepth 1 -exec rm -rf -- {{}} +") + commands.append(f"chmod 777 {quoted}") + # Reference preparation is best effort; the following transfer/test detects + # an unusable directory and retains the provider's concrete error. + await environment.exec(" && ".join(commands), user="root") diff --git a/resources_servers/terminal_bench_4/verifier.py b/resources_servers/terminal_bench_4/verifier.py new file mode 100644 index 0000000000..d3cd3c47a2 --- /dev/null +++ b/resources_servers/terminal_bench_4/verifier.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Restore task artifacts and run the official baked-in verifier.""" + +import asyncio +import json +import math +import shlex +from pathlib import Path, PurePosixPath + +from resources_servers.terminal_bench_4.task import resolve_env +from resources_servers.terminal_bench_4.transfers import download_dir, prepare_directory, upload_dir, upload_file + + +class RewardFileNotFoundError(FileNotFoundError): + pass + + +class RewardFileEmptyError(ValueError): + pass + + +class VerifierOutputParseError(ValueError): + pass + + +class VerifierTimeoutError(TimeoutError): + pass + + +def parse_reward(directory): + directory = Path(directory) + json_path, text_path = directory / "reward.json", directory / "reward.txt" + path = json_path if json_path.exists() else text_path + if not path.exists(): + raise RewardFileNotFoundError(f"No official reward file found in {directory}") + text = path.read_text() + if not text: + raise RewardFileEmptyError(f"Reward file is empty: {path}") + try: + rewards = json.loads(text) if path == json_path else {"reward": float(text)} + if not isinstance(rewards, dict): + raise ValueError("Reward JSON must be an object") + for value in rewards.values(): + if not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValueError("Official rewards must be finite numbers") + return rewards + except (ValueError, TypeError) as exc: + raise VerifierOutputParseError(f"Invalid official reward in {path}: {exc}") from exc + + +async def restore(environment, artifacts_dir): + await prepare_directory(environment, "/logs/verifier", empty=True) + shared_logs = getattr(environment, "shared_logs", None) + for artifact in environment.task.config.collected_artifacts: + host = Path(artifacts_dir) / artifact.host_path + if not host.exists(): + continue + target = artifact.source + if host.is_dir(): + await prepare_directory(environment, target, empty=True) + if shared_logs and shared_logs.restored_archive and target == "/logs/artifacts": + archive = shlex.quote(shared_logs.restored_archive) + result = await environment.main.exec( + f"tar -xzf {archive} -C /logs/artifacts; status=$?; rm -f {archive}; exit $status", + timeout_s=600, + ) + if result.return_code == 0: + continue + # Retain the normal transfer fallback if remote extraction is + # unavailable, clearing any partial extraction first. + await prepare_directory(environment, target, empty=True) + await upload_dir(environment.main, host, target) + else: + parent = str(PurePosixPath(target).parent) + if parent and parent != target: + await prepare_directory(environment, parent) + await upload_file(environment.main, host, target) + + +async def run_verifier(environment, directory, diagnostics): + settings = environment.task.config.verifier + logs = Path(directory) / "verifier" + logs.mkdir(parents=True, exist_ok=True) + + async def execute(): + await environment.exec("chmod +x /tests/test.sh", user="root") + result = await environment.exec( + "/tests/test.sh > /logs/verifier/test-stdout.txt 2>&1", + env=resolve_env(settings.env), + user=settings.user, + ) + diagnostics.append({"operation": "verifier_command", "return_code": result.return_code}) + await download_dir(environment.main, "/logs/verifier", logs) + return {"rewards": parse_reward(logs)} + + try: + return await asyncio.wait_for(execute(), timeout=settings.timeout_sec) + except TimeoutError as exc: + # Preserve diagnostics on timeout without turning partial reward files + # into a completed evaluation (the reference aborts this verifier). + try: + await download_dir(environment.main, "/logs/verifier", logs) + except Exception as download_exc: + diagnostics.append({"operation": "verifier_logs", "error": str(download_exc)}) + raise VerifierTimeoutError(f"Verifier execution timed out after {settings.timeout_sec} seconds") from exc diff --git a/responses_api_agents/miniswe_sandboxed_agent/README.md b/responses_api_agents/miniswe_sandboxed_agent/README.md new file mode 100644 index 0000000000..b67fe04792 --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/README.md @@ -0,0 +1,28 @@ +# Sandboxed mini-SWE + +Generic mini-SWE 2.1.0 `DefaultAgent` execution on an environment owned by a Gym +resources server. It uses the sandbox seed/start/verify contract. It has no SWE-bench image, patch, dataset, or grading assumptions. + +The existing synchronous mini-SWE loop uses an explicit bridge to Gym's async +Responses model client and sandbox operations. Cancellation closes pending I/O +and joins the loop before requesting verification. The worker releases its +connection; resources retain destruction ownership and the authoritative budget. + +Profile `tb4-miniswe-text-v1` uses one text-form bash action per model response. +The generic prompt defines completion as a successful command whose first output +line is `COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT`, matching mini-SWE's convention. +`step_limit=0` and `cost_limit=0` leave those caps disabled; resources bound time. +Every step persists the native mini-SWE trajectory, including observations. + +Task skills are exposed by their supplied directory. For MCP tasks, setup installs +`mcp==1.29.0` into a task-local virtual environment, discovers the declared tools, +and adds their schemas and invocation command to the prompt. The CLI supports +stdio, SSE, and streamable HTTP; calls execute inside the main sandbox so service +names retain their task-network meaning. A persistent MCP session preserves state +across calls. Image tool results become multimodal model inputs. This changes the evaluation profile +relative to native mini-SWE and must be disclosed in score comparisons. + +Use [the TB4 mini-SWE profile](../../benchmarks/terminal_bench_4/miniswe.yaml) with +[TB4 resources](../../resources_servers/terminal_bench_4/README.md). Existing +`mini_swe_agent_2` SWE-bench behavior remains unchanged. Coverage is a validation +claim, not implied by selecting this configuration. diff --git a/responses_api_agents/miniswe_sandboxed_agent/app.py b/responses_api_agents/miniswe_sandboxed_agent/app.py new file mode 100644 index 0000000000..a759b47997 --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/app.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""mini-SWE 2.1 DefaultAgent on a borrowed environment, with Gym model routing.""" + +import asyncio +import json +from pathlib import Path +from shlex import quote +from threading import Lock +from typing import Any + +from fastapi import Request +from minisweagent.agents.default import DefaultAgent +from minisweagent.environments.local import LocalEnvironment +from minisweagent.models.utils.actions_text import format_observation_messages, parse_regex_actions +from pydantic import ConfigDict, Field + +from nemo_gym.base_resources_server import BaseRunRequest +from nemo_gym.base_responses_api_agent import BaseResponsesAPIAgentConfig, SimpleResponsesAPIAgent +from nemo_gym.config_types import ModelServerRef, ResourcesServerRef +from nemo_gym.openai_utils import NeMoGymResponse, NeMoGymResponseUsage +from nemo_gym.sandbox.agent import artifact_directory, empty_response, run_borrowed +from nemo_gym.sandbox.handoff import AgentTermination, SandboxedVerifyResponse +from nemo_gym.server_utils import get_response_json, is_nemo_gym_fastapi_entrypoint, raise_for_status + + +SYSTEM = """You are an assistant operating a task environment. Respond with exactly one bash action in a +```mswea_bash_command code block. Commands run in separate shells; filesystem changes persist. +Complete the task in the environment. To finish, run only: echo COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT +After submission you cannot change the environment. Task-specific instructions take precedence.""" + + +class MiniSWESandboxedConfig(BaseResponsesAPIAgentConfig): + model_server: ModelServerRef + resources_server: ResourcesServerRef + sandbox_providers: dict[str, Any] + step_limit: int = Field(default=0, ge=0) + step_timeout_sec: int = Field(default=600, gt=0) + + +class MiniSWERunRequest(BaseRunRequest): + model_config = ConfigDict(extra="allow") + + +class WorkerBridge: + """Synchronous mini-SWE loop, asynchronous Gym I/O, explicit cancellation.""" + + def __init__(self): + self.loop = asyncio.get_running_loop() + self.closed = False + self.pending = set() + self.lock = Lock() + + def call(self, factory): + with self.lock: + if self.closed: + raise RuntimeError("Episode is closed") + future = asyncio.run_coroutine_threadsafe(factory(), self.loop) + self.pending.add(future) + try: + return future.result() + finally: + with self.lock: + self.pending.discard(future) + + def close(self): + with self.lock: + self.closed = True + pending = list(self.pending) + for future in pending: + future.cancel() + + +class GymModel: + def __init__(self, bridge, query): + self.bridge, self._query = bridge, query + + def query(self, messages): + return self.bridge.call(lambda: self._query(messages)) + + def format_message(self, **kwargs): + return kwargs + + def format_observation_messages(self, message, outputs, template_vars=None): + messages = format_observation_messages( + outputs, + observation_template="{{output.returncode}}\n{{output.output}}", + ) + for observation, output in zip(messages, outputs, strict=True): + if output.get("images"): + observation["content"] = [{"type": "input_text", "text": observation["content"]}] + [ + {"type": "input_image", "image_url": uri} for uri in output["images"] + ] + return messages + + def get_template_vars(self): + return {} + + def serialize(self): + return {"info": {"model_transport": "nemo_gym_responses"}} + + +class BorrowedEnvironment: + def __init__(self, bridge, execute): + self.bridge, self._execute = bridge, execute + + def execute(self, action): + output = self.bridge.call(lambda: self._execute(action["command"])) + LocalEnvironment._check_finished(self, output) + return output + + def get_template_vars(self): + return {"system": "Linux"} + + def serialize(self): + return {"info": {"environment_type": "gym_borrowed_sandbox"}} + + +class MiniSWESandboxedAgent(SimpleResponsesAPIAgent): + config: MiniSWESandboxedConfig + + async def responses(self, body): + raise NotImplementedError("This harness requires /run with a resources session") + + async def run(self, request: Request, body: MiniSWERunRequest) -> SandboxedVerifyResponse: + extra_instruction = "" + + async def setup(sandbox, seed): + nonlocal extra_instruction + result = await sandbox.exec("command -v setsid", user=seed.user) + if result.return_code: + raise RuntimeError("mini-SWE requires setsid for process cleanup") + if seed.skills_dir: + extra_instruction += f"\nTask skills are in {seed.skills_dir}. Read the relevant SKILL.md files.\n" + if seed.mcp_servers: + directory = artifact_directory(self.config.name, seed.session_id) + (directory / "mcp.json").write_text(json.dumps(seed.mcp_servers)) + remote = f"/tmp/{seed.session_id}-mcp" + command = f"python3 -m venv {remote} && {remote}/bin/pip -q install mcp==1.29.0 httpx-aiohttp==0.2.0" + result = await sandbox.exec(command, user=seed.user, timeout_s=seed.setup_timeout_sec) + if result.return_code: + raise RuntimeError(f"Task MCP client setup failed: {result.stderr}") + await sandbox.upload(Path(__file__).with_name("mcp_client.py"), remote + "/client.py") + await sandbox.upload(directory / "mcp.json", remote + "/servers.json") + cli = f"{remote}/bin/python {remote}/client.py" + daemon = f"echo $$ >> /tmp/{seed.session_id}.pids; exec {cli} serve" + started = await sandbox.exec( + "bash -c " + + quote( + f"setsid --fork bash -c {quote(daemon)} > {remote}/server.log 2>&1 < /dev/null; " + f"for i in $(seq 1 60); do [ -S {remote}/server.sock ] && exit 0; sleep 1; done; " + f"cat {remote}/server.log; exit 1" + ), + user=seed.user, + timeout_s=65, + ) + if started.return_code: + raise RuntimeError(f"Task MCP session setup failed: {started.stdout}") + listed = await sandbox.exec(cli + " list", user=seed.user, timeout_s=60) + if listed.return_code: + raise RuntimeError(f"Task MCP discovery failed: {listed.stderr}") + extra_instruction += ( + f"\nTask MCP tools (JSON schemas): {listed.stdout}\n" + f"Call with: {cli} call SERVER TOOL 'JSON_ARGUMENTS'.\n" + ) + + async def execute(sandbox, seed, budget): + bridge = WorkerBridge() + responses = [] + directory = artifact_directory(self.config.name, seed.session_id) + + async def query(messages): + params = body.responses_create_params.model_dump(exclude_none=True) + params["input"] = [{"role": m["role"], "content": m.get("content", "")} for m in messages] + params.pop("tools", None) + params.pop("tool_choice", None) + model_response = await self.server_client.post( + server_name=self.config.model_server.name, + url_path=self.url_path_for_run(url_path="/v1/responses", body=body), + json=params, + cookies=request.cookies, + ) + await raise_for_status(model_response) + response = NeMoGymResponse.model_validate(await get_response_json(model_response)) + responses.append(response) + content = "\n".join( + part.text + for item in response.output + if item.type == "message" + for part in item.content + if part.type == "output_text" + ) + actions = parse_regex_actions( + content, + action_regex=r"```(?:mswea_bash_command|bash)\s*\n(.*?)\n```", + format_error_template="Return exactly one bash action in a mswea_bash_command code block.", + ) + return {"role": "assistant", "content": content, "extra": {"actions": actions}} + + async def command(text): + result = await sandbox.exec( + "setsid --wait bash -c " + quote(f"echo $$ >> /tmp/{seed.session_id}.pids; " + text), + user=seed.user, + timeout_s=min(budget, self.config.step_timeout_sec), + ) + if result.error_type and result.error_type != "timeout": + raise RuntimeError(f"Sandbox execution failed: {result.error_type}") + output = (result.stdout or "") + (result.stderr or "") + images = [] + try: + tool_result = json.loads(output) + for part in tool_result.get("content", []): + if part.get("type") == "image": + images.append(f"data:{part['mimeType']};base64,{part.pop('data')}") + if images: + output = json.dumps(tool_result) + except (ValueError, AttributeError, KeyError, TypeError): + pass + return {"output": output, "returncode": result.return_code, "images": images} + + agent = DefaultAgent( + GymModel(bridge, query), + BorrowedEnvironment(bridge, command), + system_template=SYSTEM, + instance_template="{{task}}", + step_limit=self.config.step_limit, + cost_limit=0, + output_path=directory / "trajectory.json", + ) + worker = asyncio.create_task(asyncio.to_thread(agent.run, seed.instruction + extra_instruction)) + termination = AgentTermination(reason="completed") + try: + info = await asyncio.wait_for(asyncio.shield(worker), budget) + if info.get("exit_status") != "Submitted": + termination = AgentTermination(reason="nonzero_exit", detail=info.get("exit_status")) + except TimeoutError: + termination = AgentTermination(reason="timeout") + except Exception as exc: + termination = AgentTermination(reason="infrastructure_error", detail=f"{type(exc).__name__}: {exc}") + finally: + bridge.close() + # Cancel pending I/O and join the synchronous loop before verification. + await asyncio.gather(worker, return_exceptions=True) + response = empty_response(body.responses_create_params, self.config.model_server.name) + response.output = [item for part in responses for item in part.output] + response.usage = NeMoGymResponseUsage.sum_from_list([r.usage for r in responses if r.usage]) + termination.artifacts = [str(directory / "trajectory.json")] + return response, termination, {"mini_swe_trajectory": agent.serialize(), "harness_version": "2.1.0"} + + return SandboxedVerifyResponse.model_validate( + await run_borrowed(self, request, body, setup=setup, execute=execute) + ) + + +if __name__ == "__main__": + MiniSWESandboxedAgent.run_webserver() +elif is_nemo_gym_fastapi_entrypoint(__file__): + app = MiniSWESandboxedAgent.run_webserver() diff --git a/responses_api_agents/miniswe_sandboxed_agent/mcp_client.py b/responses_api_agents/miniswe_sandboxed_agent/mcp_client.py new file mode 100644 index 0000000000..128ca48363 --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/mcp_client.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Small task-local CLI for mini-SWE's bash harness; no provider credentials.""" + +import asyncio +import json +import sys +from contextlib import AsyncExitStack +from pathlib import Path + +from httpx_aiohttp import HttpxAiohttpClient +from mcp import ClientSession, StdioServerParameters +from mcp.client.sse import sse_client +from mcp.client.stdio import stdio_client +from mcp.client.streamable_http import streamablehttp_client + + +def http_client(**kwargs): + return HttpxAiohttpClient(follow_redirects=True, **kwargs) + + +async def connect(server, stack): + if server["transport"] == "stdio": + transport = stdio_client(StdioServerParameters(command=server["command"], args=server.get("args", []))) + elif server["transport"] == "sse": + transport = sse_client(server["url"], httpx_client_factory=http_client) + else: + transport = streamablehttp_client(server["url"], httpx_client_factory=http_client) + streams = await stack.enter_async_context(transport) + client = await stack.enter_async_context(ClientSession(streams[0], streams[1])) + await client.initialize() + return client + + +async def list_tools(client): + tools = [] + cursor = None + while True: + page = await client.list_tools(cursor=cursor) + tools.extend(tool.model_dump(mode="json") for tool in page.tools) + cursor = page.nextCursor + if cursor is None: + return tools + + +async def dispatch(clients, message): + if message["operation"] == "list": + return {name: await list_tools(client) for name, client in clients.items()} + return (await clients[message["server"]].call_tool(message["tool"], message["arguments"])).model_dump(mode="json") + + +async def serve(servers, socket_path): + # Keep MCP sessions alive across CLI calls (e.g. browser page state). + async with AsyncExitStack() as stack: + clients = {server["name"]: await connect(server, stack) for server in servers} + + async def handle(reader, writer): + try: + message = json.loads(await reader.readline()) + result = await dispatch(clients, message) + writer.write(json.dumps({"result": result}).encode() + b"\n") + except Exception as exc: + writer.write(json.dumps({"error": f"{type(exc).__name__}: {exc}"}).encode() + b"\n") + finally: + await writer.drain() + writer.close() + await writer.wait_closed() + + server = await asyncio.start_unix_server(handle, path=socket_path, limit=16 * 1024 * 1024) + Path(socket_path).chmod(0o600) + async with server: + await server.serve_forever() + + +async def request(socket_path, message): + reader, writer = await asyncio.open_unix_connection(socket_path, limit=16 * 1024 * 1024) + try: + writer.write(json.dumps(message).encode() + b"\n") + await writer.drain() + response = json.loads(await reader.readline()) + if "error" in response: + raise RuntimeError(response["error"]) + return response["result"] + finally: + writer.close() + await writer.wait_closed() + + +async def main(): + servers = json.loads(Path(__file__).with_name("servers.json").read_text()) + socket_path = str(Path(__file__).with_name("server.sock")) + if sys.argv[1] == "serve": + await serve(servers, socket_path) + return + if sys.argv[1] == "list": + message = {"operation": "list"} + else: + message = { + "operation": "call", + "server": sys.argv[2], + "tool": sys.argv[3], + "arguments": json.loads(sys.argv[4]), + } + result = await request(socket_path, message) + print(json.dumps(result)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/responses_api_agents/miniswe_sandboxed_agent/requirements.txt b/responses_api_agents/miniswe_sandboxed_agent/requirements.txt new file mode 100644 index 0000000000..2861fbcdb5 --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/requirements.txt @@ -0,0 +1,4 @@ +-e nemo-gym[dev,sandbox] @ ../../ +mini-swe-agent==2.1.0 +mcp==1.29.0 +httpx-aiohttp==0.2.0 diff --git a/responses_api_agents/miniswe_sandboxed_agent/tests/test_app.py b/responses_api_agents/miniswe_sandboxed_agent/tests/test_app.py new file mode 100644 index 0000000000..f825150e88 --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/tests/test_app.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from nemo_gym.sandbox import SandboxExecResult +from nemo_gym.sandbox.handoff import SandboxedSeedResponse +from nemo_gym.server_utils import ServerClient +from responses_api_agents.miniswe_sandboxed_agent import app as module + + +@pytest.mark.parametrize("with_mcp", [False, True]) +async def test_real_default_agent_loop_uses_gym_model_and_borrowed_commands(tmp_path, monkeypatch, with_mcp): + monkeypatch.chdir(tmp_path) + client = MagicMock(spec=ServerClient) + client.post = AsyncMock( + return_value=SimpleNamespace( + value={ + "id": "resp_test", + "created_at": 0, + "object": "response", + "model": "test", + "parallel_tool_calls": False, + "tool_choice": "auto", + "tools": [], + "output": [ + { + "type": "message", + "id": "msg_test", + "status": "completed", + "role": "assistant", + "content": [ + { + "type": "output_text", + "text": "```mswea_bash_command\necho COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\n```", + "annotations": [], + } + ], + } + ], + "usage": { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 13, + }, + } + ) + ) + + async def decode(r): + return r.value + + monkeypatch.setattr(module, "get_response_json", decode) + monkeypatch.setattr(module, "raise_for_status", AsyncMock()) + commands = [] + + async def execute(command, **kwargs): + commands.append(command) + if "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT" in command: + return SandboxExecResult("COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\n", "", 0) + return SandboxExecResult('{"browser": []}', "", 0) + + sandbox = SimpleNamespace(exec=execute, upload=AsyncMock()) + seed = SandboxedSeedResponse( + session_id="task", + sandbox={"provider": "cpu", "sandbox_id": "box"}, + instruction="Official task instruction", + agent_timeout_sec=5, + setup_timeout_sec=5, + skills_dir="/skills", + mcp_servers=[{"name": "browser", "transport": "streamable-http", "url": "http://sidecar/mcp"}] + if with_mcp + else [], + ) + + async def lifecycle(agent, request, body, *, setup, execute): + await setup(sandbox, seed) + response, termination, extra = await execute(sandbox, seed, 5) + return { + **body.model_dump(), + "response": response.model_dump(), + "session_id": "task", + "termination": termination.model_dump(), + "reward": 1, + "evaluation_completed": True, + **extra, + } + + monkeypatch.setattr(module, "run_borrowed", lifecycle) + server = module.MiniSWESandboxedAgent( + config=module.MiniSWESandboxedConfig( + name="agent", + host="localhost", + port=1, + entrypoint="app.py", + resources_server={"type": "resources_servers", "name": "resources"}, + model_server={"type": "responses_api_models", "name": "model"}, + sandbox_providers={}, + ), + server_client=client, + ) + result = await server.run( + SimpleNamespace(cookies={}), module.MiniSWERunRequest(responses_create_params={"input": []}) + ) + assert result.reward == 1 + assert result.response.usage.total_tokens == 13 + assert result.termination.reason == "completed" + assert any(c.startswith("setsid --wait") for c in commands) + assert client.post.await_args.kwargs["json"]["input"][1]["content"].startswith(seed.instruction) + if with_mcp: + assert any("setsid --fork" in c and "server.sock" in c for c in commands) + assert sandbox.upload.await_count == 2 diff --git a/responses_api_agents/miniswe_sandboxed_agent/tests/test_mcp.py b/responses_api_agents/miniswe_sandboxed_agent/tests/test_mcp.py new file mode 100644 index 0000000000..ddd7ddcefc --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/tests/test_mcp.py @@ -0,0 +1,177 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from contextlib import AsyncExitStack, asynccontextmanager +from pathlib import Path +from tempfile import TemporaryDirectory +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from responses_api_agents.miniswe_sandboxed_agent import mcp_client as module +from responses_api_agents.miniswe_sandboxed_agent.app import GymModel + + +@pytest.mark.parametrize("transport", ["stdio", "sse", "streamable-http"]) +async def test_transport_connections_use_declared_endpoints(monkeypatch, transport): + calls = [] + + @asynccontextmanager + async def connection(*args, **kwargs): + calls.append((args, kwargs)) + yield ("read", "write", "unused") + + client = SimpleNamespace(initialize=AsyncMock()) + + @asynccontextmanager + async def session(read, write): + assert (read, write) == ("read", "write") + yield client + + for method in ["stdio_client", "sse_client", "streamablehttp_client"]: + monkeypatch.setattr(module, method, connection) + monkeypatch.setattr(module, "ClientSession", session) + server = { + "name": "browser", + "transport": transport, + "command": "tool", + "args": ["serve"], + "url": "http://sidecar/mcp", + } + async with AsyncExitStack() as stack: + assert await module.connect(server, stack) is client + client.initialize.assert_awaited_once() + if transport == "stdio": + assert calls[0][0][0].command == "tool" + else: + assert calls[0][0][0] == server["url"] + assert calls[0][1]["httpx_client_factory"] is module.http_client + + +async def test_mcp_session_preserves_tool_state_across_cli_calls(monkeypatch): + state = [] + + async def call_tool(tool, args): + if tool == "fail": + raise ValueError("tool failed") + state.append(args["value"]) + return SimpleNamespace(model_dump=lambda **_: {"values": list(state)}) + + client = SimpleNamespace( + call_tool=call_tool, + list_tools=AsyncMock( + side_effect=[ + SimpleNamespace(tools=[SimpleNamespace(model_dump=lambda **_: {"name": "one"})], nextCursor="next"), + SimpleNamespace(tools=[SimpleNamespace(model_dump=lambda **_: {"name": "two"})], nextCursor=None), + ] + ), + ) + connect = AsyncMock(return_value=client) + monkeypatch.setattr(module, "connect", connect) + with TemporaryDirectory(dir="/tmp", prefix="tb4-mcp-") as directory: + socket = str(Path(directory) / "mcp.sock") + daemon = asyncio.create_task(module.serve([{"name": "browser"}], socket)) + try: + async with asyncio.timeout(5): + while not Path(socket).exists(): + if daemon.done(): + await daemon + await asyncio.sleep(0.01) + assert await module.request(socket, {"operation": "list"}) == { + "browser": [{"name": "one"}, {"name": "two"}] + } + for value in [1, 2]: + result = await module.request( + socket, {"operation": "call", "server": "browser", "tool": "save", "arguments": {"value": value}} + ) + assert result == {"values": [1, 2]} + connect.assert_awaited_once() + with pytest.raises(RuntimeError, match="tool failed"): + await module.request( + socket, {"operation": "call", "server": "browser", "tool": "fail", "arguments": {}} + ) + finally: + daemon.cancel() + await asyncio.gather(daemon, return_exceptions=True) + + +def test_mcp_images_are_multimodal_model_inputs(): + model = GymModel(None, None) + messages = model.format_observation_messages( + {}, [{"output": "Screenshot", "returncode": 0, "images": ["data:image/png;base64,AA=="]}] + ) + assert messages[0]["content"][1] == {"type": "input_image", "image_url": "data:image/png;base64,AA=="} + + +@pytest.mark.parametrize("operation", ["serve", "list", "call"]) +async def test_cli_dispatches_declared_tool_arguments(tmp_path, monkeypatch, capsys, operation): + import json + + monkeypatch.setattr(module, "__file__", str(tmp_path / "client.py")) + servers = [{"name": "browser", "transport": "sse", "url": "http://sidecar/mcp"}] + (tmp_path / "servers.json").write_text(json.dumps(servers)) + monkeypatch.setattr(module.sys, "argv", ["client.py", operation, "browser", "navigate", '{"url":"http://app"}']) + serve = AsyncMock() + request = AsyncMock(return_value={"content": [{"type": "text", "text": "ok"}]}) + monkeypatch.setattr(module, "serve", serve) + monkeypatch.setattr(module, "request", request) + await module.main() + if operation == "serve": + serve.assert_awaited_once_with(servers, str(tmp_path / "server.sock")) + request.assert_not_awaited() + else: + assert json.loads(capsys.readouterr().out)["content"][0]["text"] == "ok" + expected = ( + {"operation": "list"} + if operation == "list" + else {"operation": "call", "server": "browser", "tool": "navigate", "arguments": {"url": "http://app"}} + ) + request.assert_awaited_once_with(str(tmp_path / "server.sock"), expected) + + +async def test_http_transport_uses_aiohttp(): + async with module.http_client(timeout=10) as client: + assert client.follow_redirects + assert type(client).__name__ == "HttpxAiohttpClient" + + +async def test_real_mcp_stdio_server_retains_state(tmp_path): + import sys + + script = tmp_path / "server.py" + script.write_text("""from mcp.server.fastmcp import FastMCP +mcp = FastMCP("counter") +count = 0 +@mcp.tool() +def increment() -> int: + global count + count += 1 + return count +mcp.run(transport="stdio") +""") + with TemporaryDirectory(dir="/tmp", prefix="tb4-mcp-real-") as directory: + socket = str(Path(directory) / "mcp.sock") + daemon = asyncio.create_task( + module.serve( + [{"name": "counter", "transport": "stdio", "command": sys.executable, "args": [str(script)]}], socket + ) + ) + try: + async with asyncio.timeout(15): + while not Path(socket).exists(): + if daemon.done(): + await daemon + await asyncio.sleep(0.02) + schemas = await module.request(socket, {"operation": "list"}) + assert schemas["counter"][0]["name"] == "increment" + for expected in [1, 2]: + result = await module.request( + socket, {"operation": "call", "server": "counter", "tool": "increment", "arguments": {}} + ) + assert not result["isError"] + assert result["content"][0]["text"] == str(expected) + finally: + daemon.cancel() + await asyncio.gather(daemon, return_exceptions=True) diff --git a/responses_api_agents/miniswe_sandboxed_agent/tests/test_worker.py b/responses_api_agents/miniswe_sandboxed_agent/tests/test_worker.py new file mode 100644 index 0000000000..6418a01808 --- /dev/null +++ b/responses_api_agents/miniswe_sandboxed_agent/tests/test_worker.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio + +import pytest +from minisweagent.agents.default import DefaultAgent + +from responses_api_agents.miniswe_sandboxed_agent.app import BorrowedEnvironment, GymModel, WorkerBridge + + +async def test_default_agent_submits_in_borrowed_environment(tmp_path): + bridge = WorkerBridge() + commands = [] + + async def query(messages): + return {"role": "assistant", "content": "Submit", "extra": {"actions": [{"command": "submit"}]}} + + async def execute(command): + commands.append(command) + return {"output": "COMPLETE_TASK_AND_SUBMIT_FINAL_OUTPUT\nfinished", "returncode": 0} + + agent = DefaultAgent( + GymModel(bridge, query), + BorrowedEnvironment(bridge, execute), + system_template="System", + instance_template="{{task}}", + cost_limit=0, + output_path=tmp_path / "trajectory.json", + ) + result = await asyncio.to_thread(agent.run, "Generic task without SWE-bench fields") + bridge.close() + assert result["exit_status"] == "Submitted" + assert result["submission"] == "finished" + assert commands == ["submit"] + assert (tmp_path / "trajectory.json").is_file() + assert agent.messages[1]["content"] == "Generic task without SWE-bench fields" + + +async def test_cancellation_stops_worker_before_verification(): + bridge = WorkerBridge() + entered = asyncio.Event() + exited = asyncio.Event() + + async def query(): + entered.set() + try: + await asyncio.Event().wait() + finally: + exited.set() + + worker = asyncio.create_task(asyncio.to_thread(bridge.call, query)) + await entered.wait() + bridge.close() + await asyncio.gather(worker, return_exceptions=True) + await exited.wait() + with pytest.raises(RuntimeError, match="closed"): + await asyncio.to_thread(bridge.call, query) diff --git a/tests/unit_tests/test_docker_compose.py b/tests/unit_tests/test_docker_compose.py index 2ebbf21bff..d1e18d4e07 100644 --- a/tests/unit_tests/test_docker_compose.py +++ b/tests/unit_tests/test_docker_compose.py @@ -1317,3 +1317,22 @@ async def test_unresolved_yaml_shapes_fail_before_provisioning(tmp_path, documen with pytest.raises(ValueError, match="mapping|upstream"): await AsyncSandboxCompose(provider, path).start() assert provider.created == [] + + +async def test_cleanup_failure_still_attempts_volume_helper_and_transport(): + provider = Provider() + provider.aclose = AsyncMock() + group = AsyncSandboxCompose(provider, None) + broken = SimpleNamespace(stop=AsyncMock(side_effect=RuntimeError("service delete failed"))) + healthy = SimpleNamespace(stop=AsyncMock()) + helper = SimpleNamespace( + exec=AsyncMock(return_value=SimpleNamespace(return_code=1, stderr="volume failed")), stop=AsyncMock() + ) + group.services = {"broken": broken, "healthy": healthy} + group._volume_helper = helper + with pytest.raises(ExceptionGroup) as exc: + await group.stop() + assert len(exc.value.exceptions) == 2 + healthy.stop.assert_awaited_once() + helper.stop.assert_awaited_once() + provider.aclose.assert_awaited_once() diff --git a/tests/unit_tests/test_opensandbox_compose.py b/tests/unit_tests/test_opensandbox_compose.py index 912d6f430b..a7285af423 100644 --- a/tests/unit_tests/test_opensandbox_compose.py +++ b/tests/unit_tests/test_opensandbox_compose.py @@ -163,12 +163,19 @@ async def test_runtime_requirements_return_metadata_and_verify_without_remount(m provider = OpenSandboxProvider( runtime_requirements={ "capability_probes": {"SYS_PTRACE": "gdb-probe"}, + "capability_metadata": {"SYS_PTRACE": {"nemo.nvidia.com/ptrace": "true"}}, "shm_size_metadata_key": "example.test/shm", } ) assert provider.validate_runtime_requirements(cap_add=("SYS_PTRACE",), shm_size=1073741824) == { - "example.test/shm": "1073741824" + "example.test/shm": "1073741824", + "nemo.nvidia.com/ptrace": "true", } + assert provider.validate_runtime_requirements(cap_add=("SYS_PTRACE",), shm_size=None) == { + "nemo.nvidia.com/ptrace": "true" + } + assert provider.validate_runtime_requirements(cap_add=(), shm_size=1024) == {"example.test/shm": "1024"} + assert provider.validate_runtime_requirements(cap_add=(), shm_size=None) == {} execute = AsyncMock(return_value=SandboxExecResult("", "", 0)) monkeypatch.setattr(provider, "exec", execute) await provider.configure_runtime( diff --git a/tests/unit_tests/test_opensandbox_provider.py b/tests/unit_tests/test_opensandbox_provider.py index f3877eb2a2..9fe7263f7d 100644 --- a/tests/unit_tests/test_opensandbox_provider.py +++ b/tests/unit_tests/test_opensandbox_provider.py @@ -22,6 +22,7 @@ from pathlib import Path from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock import httpx import pytest @@ -40,6 +41,138 @@ TEST_REGISTRY_PASSWORD = "secret" # pragma: allowlist secret +@pytest.mark.parametrize("recovers", [True, False]) +@pytest.mark.parametrize("timeout_kind", ["client", "server", "readiness"]) +async def test_tb4_create_timeout_retries_with_jitter(monkeypatch, recovers, timeout_kind): + import tenacity + import tenacity.wait + import yaml + from opensandbox.exceptions import SandboxApiException, SandboxReadyTimeoutException + + config = yaml.safe_load(Path("benchmarks/terminal_bench_4/resources.yaml").read_text()) + environment = config["terminal_bench_4"]["resources_servers"]["terminal_bench_4"]["environment"] + provider = opensandbox_provider.OpenSandboxProvider( + create=environment["sandbox_provider"]["opensandbox"]["create"] + ) + errors = { + "client": opensandbox_provider.OpenSandboxCreateTimeoutError("create timeout"), + "server": SandboxApiException("POD_READY_TIMEOUT: BATCHSANDBOX_PENDING", status_code=504), + "readiness": SandboxReadyTimeoutException("readiness timeout"), + } + handle = opensandbox_provider.SandboxHandle(sandbox_id="ready", provider_name="opensandbox", raw=None) + provider._create_once = AsyncMock( + side_effect=[errors[timeout_kind]] * 5 + [handle if recovers else errors[timeout_kind]] + ) + sleep = AsyncMock() + monkeypatch.setattr( + opensandbox_provider, + "_require_tenacity", + lambda: ( + lambda **kwargs: tenacity.AsyncRetrying(sleep=sleep, **kwargs), + tenacity.retry_if_exception, + tenacity.stop_after_attempt, + tenacity.wait_random_exponential, + ), + ) + bounds = [] + + def midpoint(low, high): + bounds.append((low, high)) + return (low + high) / 2 + + monkeypatch.setattr(tenacity.wait.random, "uniform", midpoint) + if recovers: + assert await provider.create(SandboxSpec(image="task")) is handle + else: + with pytest.raises(type(errors[timeout_kind])): + await provider.create(SandboxSpec(image="task")) + assert provider._create_once.await_count == 6 + assert [call.args[0] for call in sleep.await_args_list] == [2.5, 5, 10, 20, 30] + assert bounds[:5] == [(0, 5), (0, 10), (0, 20), (0, 40), (0, 60)] + + +async def test_create_does_not_retry_rejected_credentials(): + from opensandbox.exceptions import SandboxApiException + + provider = opensandbox_provider.OpenSandboxProvider(create={"retries": 5}) + provider._create_once = AsyncMock(side_effect=SandboxApiException("unauthorized", status_code=401)) + with pytest.raises(SandboxApiException): + await provider.create(SandboxSpec(image="task")) + provider._create_once.assert_awaited_once() + + +@pytest.mark.parametrize("interval", [0, -1, float("nan"), float("inf")]) +def test_renewal_rejects_invalid_intervals(interval: float) -> None: + with pytest.raises(ValueError, match="renew_interval_s"): + opensandbox_provider.OpenSandboxCreateConfig(renew_interval_s=interval) + + +@pytest.mark.parametrize("ttl", [None, 1, 2]) +async def test_renewal_requires_ttl_longer_than_interval(ttl: int | None) -> None: + provider = opensandbox_provider.OpenSandboxProvider(create={"renew_interval_s": 2, "retries": 0}) + with pytest.raises(ValueError, match="longer, explicit"): + await provider.create(SandboxSpec(image="example", ttl_s=ttl)) + + +async def test_renewal_refreshes_ttl_and_stops_before_termination(fake_opensandbox_sdk: None) -> None: + provider = opensandbox_provider.OpenSandboxProvider(create={"renew_interval_s": 0.01}, probe={"command": None}) + handle = await provider.create(SandboxSpec(image="example", ttl_s=28800)) + renewed = asyncio.Event() + durations = [] + + async def renew(duration: timedelta) -> None: + durations.append(duration) + renewed.set() + + async def kill() -> None: + assert handle.sandbox_id not in provider._renewals + + handle.raw.renew = renew + handle.raw.kill = AsyncMock(side_effect=kill) + handle.raw.close = AsyncMock() + await asyncio.wait_for(renewed.wait(), timeout=1) + task = provider._renewals[handle.sandbox_id] + await provider.close(handle) + assert durations and all(duration == timedelta(seconds=28800) for duration in durations) + assert task.cancelled() + handle.raw.kill.assert_awaited_once() + handle.raw.close.assert_awaited_once() + await provider.aclose() + + +async def test_renewal_failure_surfaces_and_still_allows_cleanup(fake_opensandbox_sdk: None) -> None: + provider = opensandbox_provider.OpenSandboxProvider( + create={"renew_interval_s": 0.01}, probe={"command": None}, operations={"retries": 0} + ) + handle = await provider.create(SandboxSpec(image="example", ttl_s=28800)) + handle.raw.renew = AsyncMock(side_effect=RuntimeError("renewal rejected")) + handle.raw.kill = AsyncMock() + handle.raw.close = AsyncMock() + task = provider._renewals[handle.sandbox_id] + with pytest.raises(RuntimeError, match="renewal rejected"): + await asyncio.wait_for(asyncio.shield(task), timeout=1) + operation = AsyncMock() + with pytest.raises(RuntimeError, match="lifetime renewal failed"): + await provider._await_sdk_operation(operation, operation="get_info", sandbox_id=handle.sandbox_id, timeout_s=1) + operation.assert_not_awaited() + with pytest.raises(RuntimeError, match="lifetime renewal failed"): + await provider.close(handle) + handle.raw.kill.assert_awaited_once() + handle.raw.close.assert_awaited_once() + await provider.aclose() + + +async def test_provider_shutdown_cancels_renewal_without_terminating_sandbox(fake_opensandbox_sdk: None) -> None: + provider = opensandbox_provider.OpenSandboxProvider(create={"renew_interval_s": 10}, probe={"command": None}) + handle = await provider.create(SandboxSpec(image="example", ttl_s=28800)) + handle.raw.kill = AsyncMock() + task = provider._renewals[handle.sandbox_id] + await provider.aclose() + assert task.cancelled() + assert not provider._renewals + handle.raw.kill.assert_not_awaited() + + @dataclass(frozen=True) class FakePlatformSpec: os: str @@ -274,6 +407,31 @@ async def test_direct_create_passes_resource_requests_to_sdk_create( ) +@pytest.mark.parametrize( + ("resources", "expected"), + [ + ( + {"cpu": 4, "memory_mib": 4096, "disk_gib": 10}, + {"cpu": "4", "memory": "4096Mi", "ephemeral-storage": "10Gi"}, + ), + ({"cpu": 0.5, "memory_mib": 512}, {"cpu": "0.5", "memory": "512Mi"}), + ( + {"cpu": 4, "memory_mib": 16384, "gpu": 1, "gpu_type": "H100"}, + {"cpu": "4", "memory": "16384Mi", "gpu": "1", "gpu_type": "H100"}, + ), + ({}, {"cpu": "1", "memory": "2Gi"}), + ], +) +async def test_explicit_requests_match_each_sandbox_limits(fake_opensandbox_sdk, resources, expected): + provider = opensandbox_provider.OpenSandboxProvider(probe={"command": None}) + await provider.create( + SandboxSpec(image="image:tag", resources=resources, provider_options={"resource_requests": "limits"}) + ) + assert FakeSandbox.created_kwargs["resource"] == expected + assert FakeSandbox.created_kwargs["resource_requests"] == expected + assert FakeSandbox.created_kwargs["resource_requests"] is not FakeSandbox.created_kwargs["resource"] + + async def test_direct_create_passes_image_auth_to_sdk_create( fake_opensandbox_sdk: None, ) -> None: