Add configurable remote chat adapters

OSWorld-main/mm_agents/README.md

@@ -16,10 +16,32 @@ And those from the open-source community:
 - `QWEN`, `QWEN-VL`
 - `CogAgent`
 - `Llama3`
+- `MiniMax-M3`
+- `MiniMax-M2.7` (text-only)
 - ...
 
 In the future, we will integrate and support more foundational models to enhance digital agents, so stay tuned.
 
+### Remote Chat Configuration
+
+The prompt agent can use the two supported models through either compatible chat protocol. Set the API key and choose a region and protocol before running an agent:
+
+```bash
+export MINIMAX_API_KEY="your_api_key"
+export MINIMAX_REGION="global_en"  # or cn_zh
+export MINIMAX_PROTOCOL="openai"    # or anthropic
+python run.py --model MiniMax-M3 --observation_type screenshot
+```
+
+The region and protocol combinations select these base URLs:
+
+| Region | OpenAI-compatible | Anthropic-compatible |
+| --- | --- | --- |
+| `global_en` | `https://api.minimax.io/v1` | `https://api.minimax.io/anthropic` |
+| `cn_zh` | `https://api.minimaxi.com/v1` | `https://api.minimaxi.com/anthropic` |
+
+`MiniMax-M3` accepts text and image observations. `MiniMax-M2.7` accepts text observations, so use `--observation_type a11y_tree` with that model. To explicitly control M3 reasoning, set `MINIMAX_THINKING` to `adaptive` or `disabled`; M2.7 reasoning remains enabled by the service.
+
 ### How to use
 
 ```python

OSWorld-main/mm_agents/agent.py

@@ -23,6 +23,7 @@
 from requests.exceptions import SSLError
 
 from mm_agents.accessibility_tree_wrap.heuristic_retrieve import filter_nodes, draw_bounding_boxes
+from mm_agents.minimax import MINIMAX_MODELS, call_minimax
 from mm_agents.prompts import SYS_PROMPT_IN_SCREENSHOT_OUT_CODE, SYS_PROMPT_IN_SCREENSHOT_OUT_ACTION, \
     SYS_PROMPT_IN_A11Y_OUT_CODE, SYS_PROMPT_IN_A11Y_OUT_ACTION, \
     SYS_PROMPT_IN_BOTH_OUT_CODE, SYS_PROMPT_IN_BOTH_OUT_ACTION, \
@@ -654,6 +655,9 @@ def call_llm(self, payload):
             else:
                 return response.json()['choices'][0]['message']['content']
 
+        elif self.model in MINIMAX_MODELS:
+            return call_minimax(payload)
+
         elif self.model.startswith("claude"):
             messages = payload["messages"]
             max_tokens = payload["max_tokens"]

OSWorld-main/mm_agents/minimax.py

@@ -0,0 +1,179 @@
+"""MiniMax-compatible chat adapters for the prompt-based desktop agent."""
+
+from __future__ import annotations
+
+import os
+import re
+from typing import Any, Mapping
+
+import requests
+
+
+MINIMAX_MODELS = frozenset({"MiniMax-M3", "MiniMax-M2.7"})
+MINIMAX_TEXT_ONLY_MODELS = frozenset({"MiniMax-M2.7"})
+MINIMAX_ENDPOINTS = {
+    "global_en": {
+        "openai": "https://api.minimax.io/v1",
+        "anthropic": "https://api.minimax.io/anthropic",
+    },
+    "cn_zh": {
+        "openai": "https://api.minimaxi.com/v1",
+        "anthropic": "https://api.minimaxi.com/anthropic",
+    },
+}
+MINIMAX_PROTOCOLS = frozenset({"openai", "anthropic"})
+
+
+def _content_parts(message: Mapping[str, Any]) -> list[Mapping[str, Any]]:
+    content = message.get("content", [])
+    if isinstance(content, str):
+        return [{"type": "text", "text": content}]
+    return list(content)
+
+
+def _contains_non_text(messages: list[Mapping[str, Any]]) -> bool:
+    return any(
+        part.get("type") not in {"text"}
+        for message in messages
+        for part in _content_parts(message)
+    )
+
+
+def _to_anthropic_content(message: Mapping[str, Any]) -> list[dict[str, Any]]:
+    content = []
+    for part in _content_parts(message):
+        part_type = part.get("type")
+        if part_type == "text":
+            content.append({"type": "text", "text": part["text"]})
+        elif part_type == "image_url":
+            image_url = part["image_url"]["url"]
+            if image_url.startswith("data:"):
+                header, image_data = image_url.split(",", 1)
+                media_type = header[5:].split(";", 1)[0] or "image/png"
+                content.append(
+                    {
+                        "type": "image",
+                        "source": {
+                            "type": "base64",
+                            "media_type": media_type,
+                            "data": image_data,
+                        },
+                    }
+                )
+            else:
+                content.append(
+                    {"type": "image", "source": {"type": "url", "url": image_url}}
+                )
+        else:
+            raise ValueError(f"Unsupported MiniMax content type: {part_type}")
+    return content
+
+
+def _selected_protocol(protocol: str | None) -> str:
+    selected = protocol or os.environ.get("MINIMAX_PROTOCOL", "openai")
+    if selected not in MINIMAX_PROTOCOLS:
+        raise ValueError("MINIMAX_PROTOCOL must be 'openai' or 'anthropic'")
+    return selected
+
+
+def _selected_region(region: str | None) -> str:
+    selected = region or os.environ.get("MINIMAX_REGION", "global_en")
+    if selected not in MINIMAX_ENDPOINTS:
+        raise ValueError("MINIMAX_REGION must be 'global_en' or 'cn_zh'")
+    return selected
+
+
+def _thinking_parameter(model: str) -> dict[str, str] | None:
+    thinking = os.environ.get("MINIMAX_THINKING")
+    if not thinking:
+        return None
+    if model != "MiniMax-M3":
+        raise ValueError("MINIMAX_THINKING only applies to MiniMax-M3")
+    if thinking not in {"adaptive", "disabled"}:
+        raise ValueError("MINIMAX_THINKING must be 'adaptive' or 'disabled'")
+    return {"type": thinking}
+
+
+def build_minimax_request(
+    payload: Mapping[str, Any],
+    *,
+    protocol: str | None = None,
+    region: str | None = None,
+    api_key: str | None = None,
+) -> tuple[str, dict[str, str], dict[str, Any]]:
+    """Build a request while keeping protocol and region selection user-configurable."""
+    model = payload["model"]
+    if model not in MINIMAX_MODELS:
+        raise ValueError(f"Unsupported MiniMax model: {model}")
+
+    messages = list(payload["messages"])
+    if model in MINIMAX_TEXT_ONLY_MODELS and _contains_non_text(messages):
+        raise ValueError(f"{model} supports text input only")
+
+    selected_protocol = _selected_protocol(protocol)
+    selected_region = _selected_region(region)
+    api_key = api_key or os.environ.get("MINIMAX_API_KEY")
+    if not api_key:
+        raise ValueError("MINIMAX_API_KEY must be set")
+
+    base_url = MINIMAX_ENDPOINTS[selected_region][selected_protocol]
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json",
+    }
+    thinking = _thinking_parameter(model)
+
+    if selected_protocol == "openai":
+        request_body = dict(payload)
+        if thinking:
+            request_body["thinking"] = thinking
+        return f"{base_url}/chat/completions", headers, request_body
+
+    request_messages = []
+    system_parts = []
+    for message in messages:
+        if message["role"] == "system":
+            system_parts.extend(
+                part["text"] for part in _content_parts(message) if part.get("type") == "text"
+            )
+            continue
+        request_messages.append(
+            {"role": message["role"], "content": _to_anthropic_content(message)}
+        )
+
+    request_body = {
+        "model": model,
+        "max_tokens": payload["max_tokens"],
+        "messages": request_messages,
+        "temperature": payload["temperature"],
+        "top_p": payload["top_p"],
+    }
+    if system_parts:
+        request_body["system"] = "\n\n".join(system_parts)
+    if thinking:
+        request_body["thinking"] = thinking
+    return f"{base_url}/v1/messages", headers, request_body
+
+
+def _text_response(content: Any) -> str:
+    if isinstance(content, str):
+        text = content
+    else:
+        text = "\n".join(
+            block.get("text", "")
+            for block in content
+            if block.get("type") == "text" and block.get("text")
+        )
+    return re.sub(r"<think>.*?</think>\s*", "", text, flags=re.DOTALL).strip()
+
+
+def call_minimax(payload: Mapping[str, Any]) -> str:
+    """Send a non-streaming request and return only assistant text."""
+    protocol = _selected_protocol(None)
+    request_url, headers, request_body = build_minimax_request(payload, protocol=protocol)
+    response = requests.post(request_url, headers=headers, json=request_body, timeout=120)
+    response.raise_for_status()
+    data = response.json()
+    if protocol == "openai":
+        return _text_response(data["choices"][0]["message"]["content"])
+    return _text_response(data["content"])

OSWorld-main/tests/test_minimax.py

@@ -0,0 +1,98 @@
+import os
+import unittest
+from unittest.mock import patch
+
+from mm_agents.minimax import build_minimax_request, call_minimax
+
+
+PAYLOAD = {
+    "model": "MiniMax-M3",
+    "messages": [
+        {"role": "system", "content": [{"type": "text", "text": "Be concise."}]},
+        {"role": "user", "content": [{"type": "text", "text": "Continue."}]},
+    ],
+    "max_tokens": 100,
+    "top_p": 0.9,
+    "temperature": 0.5,
+}
+
+
+class FakeResponse:
+    def __init__(self, body):
+        self.body = body
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self.body
+
+
+class MiniMaxRequestTests(unittest.TestCase):
+    def test_openai_region_and_protocol_selection(self):
+        url, headers, body = build_minimax_request(
+            PAYLOAD, protocol="openai", region="cn_zh", api_key="test-key"
+        )
+
+        self.assertEqual(url, "https://api.minimaxi.com/v1/chat/completions")
+        self.assertEqual(headers["Authorization"], "Bearer test-key")
+        self.assertEqual(body["model"], "MiniMax-M3")
+
+    def test_anthropic_request_preserves_system_and_image_content(self):
+        payload = {
+            **PAYLOAD,
+            "messages": [
+                PAYLOAD["messages"][0],
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "Describe this."},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/png;base64,ZmFrZQ=="},
+                        },
+                    ],
+                },
+            ],
+        }
+        url, _, body = build_minimax_request(
+            payload, protocol="anthropic", region="global_en", api_key="test-key"
+        )
+
+        self.assertEqual(url, "https://api.minimax.io/anthropic/v1/messages")
+        self.assertEqual(body["system"], "Be concise.")
+        self.assertEqual(body["messages"][0]["content"][1]["type"], "image")
+
+    def test_m27_rejects_image_input(self):
+        payload = {
+            **PAYLOAD,
+            "model": "MiniMax-M2.7",
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "data:image/png;base64,ZmFrZQ=="},
+                        }
+                    ],
+                }
+            ],
+        }
+
+        with self.assertRaisesRegex(ValueError, "text input only"):
+            build_minimax_request(payload, api_key="test-key")
+
+    @patch.dict(os.environ, {"MINIMAX_API_KEY": "test-key", "MINIMAX_PROTOCOL": "openai"})
+    @patch("mm_agents.minimax.requests.post")
+    def test_response_text_excludes_reasoning_tags(self, post):
+        post.return_value = FakeResponse(
+            {"choices": [{"message": {"content": "<think>internal</think>DONE"}}]}
+        )
+
+        self.assertEqual(call_minimax(PAYLOAD), "DONE")
+        self.assertEqual(post.call_args.kwargs["timeout"], 120)
+
+
+if __name__ == "__main__":
+    unittest.main()