From 2ec1a292644afdb921f35e16b31ee67bc1b5b8f5 Mon Sep 17 00:00:00 2001
From: octo-patch <266937838+octo-patch@users.noreply.github.com>
Date: Thu, 16 Jul 2026 20:16:07 +0800
Subject: [PATCH] Add configurable remote chat adapters
---
OSWorld-main/mm_agents/README.md | 22 ++++
OSWorld-main/mm_agents/agent.py | 4 +
OSWorld-main/mm_agents/minimax.py | 179 +++++++++++++++++++++++++++++
OSWorld-main/tests/test_minimax.py | 98 ++++++++++++++++
4 files changed, 303 insertions(+)
create mode 100644 OSWorld-main/mm_agents/minimax.py
create mode 100644 OSWorld-main/tests/test_minimax.py
diff --git a/OSWorld-main/mm_agents/README.md b/OSWorld-main/mm_agents/README.md
index e096bf2..cce71f7 100644
--- a/OSWorld-main/mm_agents/README.md
+++ b/OSWorld-main/mm_agents/README.md
@@ -16,10 +16,32 @@ And those from the open-source community:
- `QWEN`, `QWEN-VL`
- `CogAgent`
- `Llama3`
+- `MiniMax-M3`
+- `MiniMax-M2.7` (text-only)
- ...
In the future, we will integrate and support more foundational models to enhance digital agents, so stay tuned.
+### Remote Chat Configuration
+
+The prompt agent can use the two supported models through either compatible chat protocol. Set the API key and choose a region and protocol before running an agent:
+
+```bash
+export MINIMAX_API_KEY="your_api_key"
+export MINIMAX_REGION="global_en" # or cn_zh
+export MINIMAX_PROTOCOL="openai" # or anthropic
+python run.py --model MiniMax-M3 --observation_type screenshot
+```
+
+The region and protocol combinations select these base URLs:
+
+| Region | OpenAI-compatible | Anthropic-compatible |
+| --- | --- | --- |
+| `global_en` | `https://api.minimax.io/v1` | `https://api.minimax.io/anthropic` |
+| `cn_zh` | `https://api.minimaxi.com/v1` | `https://api.minimaxi.com/anthropic` |
+
+`MiniMax-M3` accepts text and image observations. `MiniMax-M2.7` accepts text observations, so use `--observation_type a11y_tree` with that model. To explicitly control M3 reasoning, set `MINIMAX_THINKING` to `adaptive` or `disabled`; M2.7 reasoning remains enabled by the service.
+
### How to use
```python
diff --git a/OSWorld-main/mm_agents/agent.py b/OSWorld-main/mm_agents/agent.py
index fd5c355..efc5503 100644
--- a/OSWorld-main/mm_agents/agent.py
+++ b/OSWorld-main/mm_agents/agent.py
@@ -23,6 +23,7 @@
from requests.exceptions import SSLError
from mm_agents.accessibility_tree_wrap.heuristic_retrieve import filter_nodes, draw_bounding_boxes
+from mm_agents.minimax import MINIMAX_MODELS, call_minimax
from mm_agents.prompts import SYS_PROMPT_IN_SCREENSHOT_OUT_CODE, SYS_PROMPT_IN_SCREENSHOT_OUT_ACTION, \
SYS_PROMPT_IN_A11Y_OUT_CODE, SYS_PROMPT_IN_A11Y_OUT_ACTION, \
SYS_PROMPT_IN_BOTH_OUT_CODE, SYS_PROMPT_IN_BOTH_OUT_ACTION, \
@@ -654,6 +655,9 @@ def call_llm(self, payload):
else:
return response.json()['choices'][0]['message']['content']
+ elif self.model in MINIMAX_MODELS:
+ return call_minimax(payload)
+
elif self.model.startswith("claude"):
messages = payload["messages"]
max_tokens = payload["max_tokens"]
diff --git a/OSWorld-main/mm_agents/minimax.py b/OSWorld-main/mm_agents/minimax.py
new file mode 100644
index 0000000..fbcf0da
--- /dev/null
+++ b/OSWorld-main/mm_agents/minimax.py
@@ -0,0 +1,179 @@
+"""MiniMax-compatible chat adapters for the prompt-based desktop agent."""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Mapping
+
+import requests
+
+
+MINIMAX_MODELS = frozenset({"MiniMax-M3", "MiniMax-M2.7"})
+MINIMAX_TEXT_ONLY_MODELS = frozenset({"MiniMax-M2.7"})
+MINIMAX_ENDPOINTS = {
+ "global_en": {
+ "openai": "https://api.minimax.io/v1",
+ "anthropic": "https://api.minimax.io/anthropic",
+ },
+ "cn_zh": {
+ "openai": "https://api.minimaxi.com/v1",
+ "anthropic": "https://api.minimaxi.com/anthropic",
+ },
+}
+MINIMAX_PROTOCOLS = frozenset({"openai", "anthropic"})
+
+
+def _content_parts(message: Mapping[str, Any]) -> list[Mapping[str, Any]]:
+ content = message.get("content", [])
+ if isinstance(content, str):
+ return [{"type": "text", "text": content}]
+ return list(content)
+
+
+def _contains_non_text(messages: list[Mapping[str, Any]]) -> bool:
+ return any(
+ part.get("type") not in {"text"}
+ for message in messages
+ for part in _content_parts(message)
+ )
+
+
+def _to_anthropic_content(message: Mapping[str, Any]) -> list[dict[str, Any]]:
+ content = []
+ for part in _content_parts(message):
+ part_type = part.get("type")
+ if part_type == "text":
+ content.append({"type": "text", "text": part["text"]})
+ elif part_type == "image_url":
+ image_url = part["image_url"]["url"]
+ if image_url.startswith("data:"):
+ header, image_data = image_url.split(",", 1)
+ media_type = header[5:].split(";", 1)[0] or "image/png"
+ content.append(
+ {
+ "type": "image",
+ "source": {
+ "type": "base64",
+ "media_type": media_type,
+ "data": image_data,
+ },
+ }
+ )
+ else:
+ content.append(
+ {"type": "image", "source": {"type": "url", "url": image_url}}
+ )
+ else:
+ raise ValueError(f"Unsupported MiniMax content type: {part_type}")
+ return content
+
+
+def _selected_protocol(protocol: str | None) -> str:
+ selected = protocol or os.environ.get("MINIMAX_PROTOCOL", "openai")
+ if selected not in MINIMAX_PROTOCOLS:
+ raise ValueError("MINIMAX_PROTOCOL must be 'openai' or 'anthropic'")
+ return selected
+
+
+def _selected_region(region: str | None) -> str:
+ selected = region or os.environ.get("MINIMAX_REGION", "global_en")
+ if selected not in MINIMAX_ENDPOINTS:
+ raise ValueError("MINIMAX_REGION must be 'global_en' or 'cn_zh'")
+ return selected
+
+
+def _thinking_parameter(model: str) -> dict[str, str] | None:
+ thinking = os.environ.get("MINIMAX_THINKING")
+ if not thinking:
+ return None
+ if model != "MiniMax-M3":
+ raise ValueError("MINIMAX_THINKING only applies to MiniMax-M3")
+ if thinking not in {"adaptive", "disabled"}:
+ raise ValueError("MINIMAX_THINKING must be 'adaptive' or 'disabled'")
+ return {"type": thinking}
+
+
+def build_minimax_request(
+ payload: Mapping[str, Any],
+ *,
+ protocol: str | None = None,
+ region: str | None = None,
+ api_key: str | None = None,
+) -> tuple[str, dict[str, str], dict[str, Any]]:
+ """Build a request while keeping protocol and region selection user-configurable."""
+ model = payload["model"]
+ if model not in MINIMAX_MODELS:
+ raise ValueError(f"Unsupported MiniMax model: {model}")
+
+ messages = list(payload["messages"])
+ if model in MINIMAX_TEXT_ONLY_MODELS and _contains_non_text(messages):
+ raise ValueError(f"{model} supports text input only")
+
+ selected_protocol = _selected_protocol(protocol)
+ selected_region = _selected_region(region)
+ api_key = api_key or os.environ.get("MINIMAX_API_KEY")
+ if not api_key:
+ raise ValueError("MINIMAX_API_KEY must be set")
+
+ base_url = MINIMAX_ENDPOINTS[selected_region][selected_protocol]
+ headers = {
+ "Authorization": f"Bearer {api_key}",
+ "Content-Type": "application/json",
+ }
+ thinking = _thinking_parameter(model)
+
+ if selected_protocol == "openai":
+ request_body = dict(payload)
+ if thinking:
+ request_body["thinking"] = thinking
+ return f"{base_url}/chat/completions", headers, request_body
+
+ request_messages = []
+ system_parts = []
+ for message in messages:
+ if message["role"] == "system":
+ system_parts.extend(
+ part["text"] for part in _content_parts(message) if part.get("type") == "text"
+ )
+ continue
+ request_messages.append(
+ {"role": message["role"], "content": _to_anthropic_content(message)}
+ )
+
+ request_body = {
+ "model": model,
+ "max_tokens": payload["max_tokens"],
+ "messages": request_messages,
+ "temperature": payload["temperature"],
+ "top_p": payload["top_p"],
+ }
+ if system_parts:
+ request_body["system"] = "\n\n".join(system_parts)
+ if thinking:
+ request_body["thinking"] = thinking
+ return f"{base_url}/v1/messages", headers, request_body
+
+
+def _text_response(content: Any) -> str:
+ if isinstance(content, str):
+ text = content
+ else:
+ text = "\n".join(
+ block.get("text", "")
+ for block in content
+ if block.get("type") == "text" and block.get("text")
+ )
+ return re.sub(r".*?\s*", "", text, flags=re.DOTALL).strip()
+
+
+def call_minimax(payload: Mapping[str, Any]) -> str:
+ """Send a non-streaming request and return only assistant text."""
+ protocol = _selected_protocol(None)
+ request_url, headers, request_body = build_minimax_request(payload, protocol=protocol)
+ response = requests.post(request_url, headers=headers, json=request_body, timeout=120)
+ response.raise_for_status()
+ data = response.json()
+ if protocol == "openai":
+ return _text_response(data["choices"][0]["message"]["content"])
+ return _text_response(data["content"])
diff --git a/OSWorld-main/tests/test_minimax.py b/OSWorld-main/tests/test_minimax.py
new file mode 100644
index 0000000..deee3ca
--- /dev/null
+++ b/OSWorld-main/tests/test_minimax.py
@@ -0,0 +1,98 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from mm_agents.minimax import build_minimax_request, call_minimax
+
+
+PAYLOAD = {
+ "model": "MiniMax-M3",
+ "messages": [
+ {"role": "system", "content": [{"type": "text", "text": "Be concise."}]},
+ {"role": "user", "content": [{"type": "text", "text": "Continue."}]},
+ ],
+ "max_tokens": 100,
+ "top_p": 0.9,
+ "temperature": 0.5,
+}
+
+
+class FakeResponse:
+ def __init__(self, body):
+ self.body = body
+
+ def raise_for_status(self):
+ return None
+
+ def json(self):
+ return self.body
+
+
+class MiniMaxRequestTests(unittest.TestCase):
+ def test_openai_region_and_protocol_selection(self):
+ url, headers, body = build_minimax_request(
+ PAYLOAD, protocol="openai", region="cn_zh", api_key="test-key"
+ )
+
+ self.assertEqual(url, "https://api.minimaxi.com/v1/chat/completions")
+ self.assertEqual(headers["Authorization"], "Bearer test-key")
+ self.assertEqual(body["model"], "MiniMax-M3")
+
+ def test_anthropic_request_preserves_system_and_image_content(self):
+ payload = {
+ **PAYLOAD,
+ "messages": [
+ PAYLOAD["messages"][0],
+ {
+ "role": "user",
+ "content": [
+ {"type": "text", "text": "Describe this."},
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,ZmFrZQ=="},
+ },
+ ],
+ },
+ ],
+ }
+ url, _, body = build_minimax_request(
+ payload, protocol="anthropic", region="global_en", api_key="test-key"
+ )
+
+ self.assertEqual(url, "https://api.minimax.io/anthropic/v1/messages")
+ self.assertEqual(body["system"], "Be concise.")
+ self.assertEqual(body["messages"][0]["content"][1]["type"], "image")
+
+ def test_m27_rejects_image_input(self):
+ payload = {
+ **PAYLOAD,
+ "model": "MiniMax-M2.7",
+ "messages": [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": "data:image/png;base64,ZmFrZQ=="},
+ }
+ ],
+ }
+ ],
+ }
+
+ with self.assertRaisesRegex(ValueError, "text input only"):
+ build_minimax_request(payload, api_key="test-key")
+
+ @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key", "MINIMAX_PROTOCOL": "openai"})
+ @patch("mm_agents.minimax.requests.post")
+ def test_response_text_excludes_reasoning_tags(self, post):
+ post.return_value = FakeResponse(
+ {"choices": [{"message": {"content": "internalDONE"}}]}
+ )
+
+ self.assertEqual(call_minimax(PAYLOAD), "DONE")
+ self.assertEqual(post.call_args.kwargs["timeout"], 120)
+
+
+if __name__ == "__main__":
+ unittest.main()