From db3f18422ea19633d0893cbf08ce36bf7040221b Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Tue, 10 Mar 2026 09:07:21 +0000 Subject: [PATCH 01/14] improve reasoning parser --- lmdeploy/serve/openai/api_server.py | 32 +-- .../serve/openai/reasoning_parser/__init__.py | 13 +- .../deepseek_r1_reasoning_parser.py | 145 ++--------- .../qwen_qwq_reasoning_parser.py | 138 +---------- .../reasoning_parser/reasoning_parser.py | 228 ++++++++++++++++-- .../openai/tool_parser/internlm2_parser.py | 7 +- .../serve/openai/tool_parser/llama3_parser.py | 7 +- .../openai/tool_parser/qwen2d5_parser.py | 7 +- .../serve/openai/tool_parser/qwen3_parser.py | 8 +- .../openai/tool_parser/qwen3coder_parser.py | 7 +- .../serve/openai/tool_parser/tool_parser.py | 16 +- 11 files changed, 282 insertions(+), 326 deletions(-) diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 3e37caffe5..e0994a2e26 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -41,7 +41,8 @@ GenerateReqInput, GenerateReqMetaOutput, GenerateReqOutput, LogProbs, ModelCard, ModelList, ModelPermission, PoolingRequest, PoolingResponse, TopLogprob, UpdateParamsRequest, UsageInfo) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser, ReasoningParserManager +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import (ReasoningParser, ReasoningParserManager, + get_streaming_state) from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParser, ToolParserManager from lmdeploy.serve.utils.server_utils import validate_json_request from lmdeploy.tokenizer import DetokenizeState, Tokenizer @@ -505,13 +506,11 @@ def create_stream_response_json(index: int, return response_json async def completion_stream_generator() -> AsyncGenerator[str, None]: - previous_text = '' - current_text = '' - previous_token_ids = [] - current_token_ids = [] - delta_token_ids = [] has_parser = VariableInterface.tool_parser is not None or VariableInterface.reasoning_parser is not None streaming_tools = False + # Shared state for streaming parsers (previous/current text & token ids) + if has_parser: + parser_state = get_streaming_state(request) async for res in result_generator: logprobs, usage = None, None if gen_logprobs and res.logprobs: @@ -534,19 +533,12 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: else: delta_message = DeltaMessage(role='assistant', content=res.response) if has_parser: - current_text = current_text + res.response - current_token_ids = current_token_ids + delta_token_ids + parser_state.update(res.response, delta_token_ids) if request.tool_choice != 'none' and VariableInterface.tool_parser is not None: if res.finish_reason == 'stop' and streaming_tools is True: res.finish_reason = 'tool_calls' tool_delta = VariableInterface.tool_parser.extract_tool_calls_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_message.content, - previous_token_ids=previous_token_ids, - current_token_ids=current_token_ids, - delta_token_ids=delta_token_ids, - request=request) + delta_text=delta_message.content, delta_token_ids=delta_token_ids, request=request) if tool_delta is not None: delta_message.tool_calls = tool_delta.tool_calls delta_message.content = tool_delta.content @@ -557,18 +549,12 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') if VariableInterface.reasoning_parser is not None and enable_thinking is not False: reasoning_delta = VariableInterface.reasoning_parser.extract_reasoning_content_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_message.content or '', - previous_token_ids=previous_token_ids, - current_token_ids=current_token_ids, - delta_token_ids=delta_token_ids) + delta_text=delta_message.content or '', delta_token_ids=delta_token_ids, request=request) if reasoning_delta is not None: delta_message.reasoning_content = reasoning_delta.reasoning_content delta_message.content = reasoning_delta.content if has_parser: - previous_text = current_text - previous_token_ids = current_token_ids + parser_state.step() if request.return_token_ids: delta_message.gen_tokens = delta_token_ids response_json = create_stream_response_json(index=0, diff --git a/lmdeploy/serve/openai/reasoning_parser/__init__.py b/lmdeploy/serve/openai/reasoning_parser/__init__.py index 09d621a252..c396a8b3ed 100644 --- a/lmdeploy/serve/openai/reasoning_parser/__init__.py +++ b/lmdeploy/serve/openai/reasoning_parser/__init__.py @@ -1,6 +1,15 @@ # Copyright (c) OpenMMLab. All rights reserved. from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser from .qwen_qwq_reasoning_parser import QwenQwQReasoningParser -from .reasoning_parser import ReasoningParser, ReasoningParserManager +from .reasoning_parser import (ReasoningParser, ReasoningParserManager, StreamingParserState, ThinkingReasoningParser, + get_streaming_state) -__all__ = ['ReasoningParser', 'ReasoningParserManager', 'DeepSeekR1ReasoningParser', 'QwenQwQReasoningParser'] +__all__ = [ + 'ReasoningParser', + 'ReasoningParserManager', + 'StreamingParserState', + 'ThinkingReasoningParser', + 'get_streaming_state', + 'DeepSeekR1ReasoningParser', + 'QwenQwQReasoningParser', +] diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py index a6b7e3a602..ca9dbaa67e 100644 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py @@ -1,140 +1,25 @@ # Copyright (c) OpenMMLab. All rights reserved. -# modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/reasoning_parsers -import re -from typing import Optional, Sequence, Tuple, Union - -from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage - -from .reasoning_parser import ReasoningParser, ReasoningParserManager +from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser @ReasoningParserManager.register_module(name='deepseek-r1') -class DeepSeekR1ReasoningParser(ReasoningParser): +class DeepSeekR1ReasoningParser(ThinkingReasoningParser): """Reasoning parser for DeepSeek R1 model. - The DeepSeek R1 model uses ... tokens to denote reasoning text. This parser extracts the reasoning - content from the model output. + Uses ... tokens. When the end tag is missing in + non-streaming mode, the entire output is treated as reasoning content + (DeepSeek R1 may omit the start tag). + + Ref: https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f """ + start_token = '' + end_token = '' + strip_newlines = False + on_missing_end_tag = 'reasoning' + def __init__(self, tokenizer: object): super().__init__(tokenizer) - self.think_start_token = '' - self.think_end_token = '' - - self.reasoning_regex = re.compile(rf'{self.think_start_token}(.*?){self.think_end_token}', re.DOTALL) - - if not self.model_tokenizer: - raise ValueError('The model tokenizer must be passed to the ReasoningParser ' - 'constructor during construction.') - - self.think_start_token_id = self.vocab.get(self.think_start_token) - self.think_end_token_id = self.vocab.get(self.think_end_token) - if (self.think_start_token_id is None or self.think_end_token_id is None): - raise RuntimeError('DeepSeek R1 reasoning parser could not locate think start/end ' - 'tokens in the tokenizer!') - - def extract_reasoning_content_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - **kwargs, - ) -> Union[DeltaMessage, None]: - """Instance method that should be implemented for extracting reasoning - from an incomplete response; for use when handling reasoning calls and - streaming. - - Has to be an instance method because it requires state - the current tokens/diffs, but also the information - about what has previously been parsed and extracted (see constructor) - """ - # Skip single special tokens - if len(delta_token_ids) == 1: - if delta_token_ids[0] == self.think_end_token_id: - return DeltaMessage(content='') - elif delta_token_ids[0] == self.think_start_token_id: - return None - - # Check if is present in previous or delta. - # Keep compatibility with models that don't generate tokens. - if self.think_start_token_id in previous_token_ids: - if self.think_end_token_id in delta_token_ids: - # in previous, in delta, - # extract reasoning content - end_index = delta_text.find(self.think_end_token) - reasoning_content = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self.think_end_token_id in previous_token_ids: - # in previous, in previous, - return DeltaMessage(content=delta_text) - else: - # in previous, no in previous or delta, - # reasoning content continues - return DeltaMessage(reasoning_content=delta_text) - elif self.think_start_token_id in delta_token_ids: - if self.think_end_token_id in delta_token_ids: - # in delta, in delta, extract reasoning content - start_index = delta_text.find(self.think_start_token) - end_index = delta_text.find(self.think_end_token) - reasoning_content = delta_text[start_index + len(self.think_start_token):end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - else: - # in delta, no in delta, - # reasoning content continues - return DeltaMessage(reasoning_content=delta_text) - else: - # No in previous or delta, also need to check for . - # Because the model may have generated without - # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self.think_end_token_id in delta_token_ids: - # in delta with more tokens, - # extract reasoning content and content - end_index = delta_text.find(self.think_end_token) - reasoning_content = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self.think_end_token_id in previous_token_ids: - # in previous, thinking content ends - return DeltaMessage(content=delta_text) - else: - # no in previous or delta, reasoning content continues - return DeltaMessage(reasoning_content=delta_text) - - def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, - **kwargs) -> Tuple[Optional[str], Optional[str]]: - """Extract reasoning content from a complete model-generated string. - - Used for non-streaming responses where we have the entire model response - available before sending to the client. - - Args: - model_output (str): The model-generated string to extract reasoning content from. - request (ChatCompletionRequest): he request object that was used to generate the model_output. - - Returns: - reasoning_content (str | None): The reasoning content. - final_output (str | None): The content. - """ - # DeepSeek R1 doesn't generate now. - # Thus we assume the reasoning content is always at the start. - # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self.think_end_token not in model_output: - return model_output, None - else: - # Add a start token if it's missing to keep compatibility. - if self.think_start_token not in model_output: - model_output = f'{self.think_start_token}{model_output}' - # Use a regex to find the reasoning content - reasoning_content = self.reasoning_regex.findall(model_output)[0] - - end_index = len(f'{self.think_start_token}{reasoning_content}{self.think_end_token}') - final_output = model_output[end_index:] - - if len(final_output) == 0: - return reasoning_content, None - - return reasoning_content, final_output + if self.start_token_id is None or self.end_token_id is None: + raise RuntimeError('DeepSeek R1 reasoning parser could not locate ' + 'think start/end tokens in the tokenizer!') diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_qwq_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_qwq_reasoning_parser.py index 3d5b792dc1..82866ad52c 100644 --- a/lmdeploy/serve/openai/reasoning_parser/qwen_qwq_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/qwen_qwq_reasoning_parser.py @@ -1,134 +1,18 @@ # Copyright (c) OpenMMLab. All rights reserved. -import re -from typing import Optional, Sequence, Tuple, Union - -from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage - -from .reasoning_parser import ReasoningParser, ReasoningParserManager +from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser @ReasoningParserManager.register_module(name=['qwen-qwq', 'intern-s1']) -class QwenQwQReasoningParser(ReasoningParser): - """Reasoning parser for Qwen QwQ model. +class QwenQwQReasoningParser(ThinkingReasoningParser): + """Reasoning parser for Qwen QwQ / Qwen3 / InternLM-S1 models. - The Qwen QwQ model uses ... tokens to denote reasoning text. This parser extracts the reasoning - content from the model output. + Uses ... tokens. When the end tag is missing in + non-streaming mode, the entire output is treated as normal content + (not reasoning). Leading/trailing newlines in reasoning content are + stripped. """ - def __init__(self, tokenizer: object): - super().__init__(tokenizer) - self.think_start_token = '' - self.think_end_token = '' - - self.reasoning_regex = re.compile(rf'{self.think_start_token}(.*?){self.think_end_token}', re.DOTALL) - - if not self.model_tokenizer: - raise ValueError('The model tokenizer must be passed to the ReasoningParser ' - 'constructor during construction.') - - def extract_reasoning_content_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - **kwargs, - ) -> Union[DeltaMessage, None]: - """Instance method that should be implemented for extracting reasoning - from an incomplete response; for use when handling reasoning calls and - streaming. - - Has to be an instance method because it requires state - the current tokens/diffs, but also the information - about what has previously been parsed and extracted (see constructor) - """ - # Skip single special tokens - if delta_text == self.think_end_token or delta_text == self.think_start_token: - return DeltaMessage(content='') - - # Check if is present in previous or delta. - # Keep compatibility with models that don't generate tokens. - if self.think_start_token in previous_text: - if self.think_end_token in delta_text: - # in previous, in delta, - # extract reasoning content - end_index = delta_text.find(self.think_end_token) - reasoning_content = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self.think_end_token in previous_text: - # in previous, in previous, - return DeltaMessage(content=delta_text) - else: - # in previous, no in previous or delta, - # reasoning content continues - return DeltaMessage(reasoning_content=delta_text) - elif self.think_start_token in delta_text: - if self.think_end_token in delta_text: - # in delta, in delta, extract reasoning content - start_index = delta_text.find(self.think_start_token) - end_index = delta_text.find(self.think_end_token) - reasoning_content = delta_text[start_index + len(self.think_start_token):end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - else: - # in delta, no in delta, - # reasoning content continues - return DeltaMessage(reasoning_content=delta_text) - else: - # No in previous or delta, also need to check for . - # Because the model may have generated without - # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self.think_end_token in delta_text: - # in delta with more tokens, - # extract reasoning content and content - end_index = delta_text.find(self.think_end_token) - reasoning_content = delta_text[:end_index] - content = delta_text[end_index + len(self.think_end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self.think_end_token in previous_text: - # in previous, thinking content ends - return DeltaMessage(content=delta_text) - else: - # no in previous or delta, reasoning content continues - return DeltaMessage(reasoning_content=delta_text) - - def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, - **kwargs) -> Tuple[Optional[str], Optional[str]]: - """Extract reasoning content from a complete model-generated string. - - Used for non-streaming responses where we have the entire model response - available before sending to the client. - - Args: - model_output (str): The model-generated string to extract reasoning content from. - request (ChatCompletionRequest): he request object that was used to generate the model_output. - - Returns: - reasoning_content (str | None): The reasoning content. - final_output (str | None): The content. - """ - # DeepSeek R1 doesn't generate now. - # Thus we assume the reasoning content is always at the start. - # Ref https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self.think_end_token not in model_output: - # for qwen3 model, the reasoning content is wrapped by xml tags - return None, model_output - # Add a start token if it's missing to keep compatibility. - if self.think_start_token not in model_output: - model_output = f'{self.think_start_token}{model_output}' - # Use a regex to find the reasoning content - reasoning_content = self.reasoning_regex.findall(model_output)[0] - - end_index = len(f'{self.think_start_token}{reasoning_content}{self.think_end_token}') - final_output = model_output[end_index:] - if reasoning_content.startswith('\n'): - reasoning_content = reasoning_content[1:] - if reasoning_content.endswith('\n'): - reasoning_content = reasoning_content[:-1] - - if len(final_output) == 0: - return reasoning_content, None - - return reasoning_content, final_output + start_token = '' + end_token = '' + strip_newlines = True + on_missing_end_tag = 'content' diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index f224dba0a5..9a6c5d90d1 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -1,7 +1,8 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/reasoning_parsers +from dataclasses import dataclass, field from functools import cached_property -from typing import Dict, Optional, Sequence, Tuple, Union +from typing import Sequence from mmengine import Registry @@ -10,51 +11,242 @@ ReasoningParserManager = Registry('reasoning_parser', locations=['lmdeploy.serve.openai.reasoning_parser']) +@dataclass +class StreamingParserState: + """Shared state for streaming parsing, attached to a request object. + + Both reasoning parsers and tool parsers read/write the same state so that text accumulated by the streaming loop is + available to all parsers without duplication. + """ + previous_text: str = '' + current_text: str = '' + previous_token_ids: list[int] = field(default_factory=list) + current_token_ids: list[int] = field(default_factory=list) + + def update(self, delta_text: str, delta_token_ids: Sequence[int]) -> None: + """Accumulate new delta into current_text / current_token_ids.""" + self.current_text += delta_text + self.current_token_ids = self.current_token_ids + list(delta_token_ids) + + def step(self) -> None: + """Advance: copy current -> previous (call at end of each iteration).""" + self.previous_text = self.current_text + self.previous_token_ids = list(self.current_token_ids) + + +def get_streaming_state(request: object) -> StreamingParserState: + """Get or create a StreamingParserState on the request object.""" + state = getattr(request, '_streaming_parser_state', None) + if state is None: + state = StreamingParserState() + setattr(request, '_streaming_parser_state', state) + return state + + class ReasoningParser: + """Abstract base class for reasoning content parsers.""" def __init__(self, tokenizer: object): self.model_tokenizer = tokenizer @cached_property - def vocab(self) -> Dict[str, int]: + def vocab(self) -> dict[str, int]: # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab # whereas all tokenizers have .get_vocab() return self.model_tokenizer.get_vocab() def extract_reasoning_content_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], + request: object, **kwargs, - ) -> Union[DeltaMessage, None]: - """Instance method that should be implemented for extracting reasoning - from an incomplete response; for use when handling reasoning calls and - streaming. + ) -> DeltaMessage | None: + """Extract reasoning content from an incomplete (streaming) response. + + Args: + delta_text: The new text chunk (may have been modified by the tool + parser before being passed here). + delta_token_ids: The new token ids for this chunk. + request: The request object; a ``StreamingParserState`` is attached + to it via ``get_streaming_state(request)`` so that previous / + current text and token ids are available. - Has to be an instance method because it requires state - the current tokens/diffs, but also the information - about what has previously been parsed and extracted (see constructor) + Returns a DeltaMessage with reasoning_content and/or content fields, + or None if the delta should be skipped. """ raise NotImplementedError('ReasoningParser.extract_reasoning_content_streaming ' 'has not been implemented!') - def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, - **kwargs) -> Tuple[Optional[str], Optional[str]]: + def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, **kwargs) -> tuple[str, str]: """Extract reasoning content from a complete model-generated string. Used for non-streaming responses where we have the entire model response available before sending to the client. Args: - model_output (str): The model-generated string to extract reasoning content from. - request (ChatCompletionRequest): he request object that was used to generate the model_output. + model_output: The model-generated string to extract reasoning content from. + request: The request object that was used to generate the model_output. Returns: - reasoning_content (str | None): The reasoning content. - final_output (str | None): The content. + A tuple of (reasoning_content, final_output). Either may be None. """ raise NotImplementedError('ReasoningParser.extract_reasoning_content ' 'has not been implemented!') + + +class ThinkingReasoningParser(ReasoningParser): + """Base class for reasoning parsers that use ... style tags. + + Subclasses only need to set `start_token`, `end_token`, and optionally + override `strip_newlines` and `on_missing_start_tag` to customize behavior. + + This parser uses a two-step detection strategy (inspired by vllm): + 1. First check token_ids (fast integer comparison) to determine whether + the start/end tags are present. + 2. Only when confirmed, use str.find() to locate exact positions for + slicing. + If the tokenizer does not have single-token representations for the tags, + it falls back to string-based detection automatically. + """ + + # Subclasses should set these + start_token: str = '' + end_token: str = '' + + # Whether to strip leading/trailing newlines from reasoning content + # in non-streaming extraction. + strip_newlines: bool = False + + # Behavior when end_token is not found in non-streaming extraction: + # 'reasoning' -> treat entire output as reasoning (DeepSeek R1 behavior) + # 'content' -> treat entire output as content (QwQ/Qwen3 behavior) + on_missing_end_tag: str = 'content' + + def __init__(self, tokenizer: object): + super().__init__(tokenizer) + + if not self.model_tokenizer: + raise ValueError('The model tokenizer must be passed to the ' + 'ReasoningParser constructor during construction.') + + # Try to resolve single token ids for fast detection. + # If the tokenizer doesn't have them as single tokens, fall back to + # string-based detection (token ids will be None). + self.start_token_id: int = self.vocab.get(self.start_token) + self.end_token_id: int = self.vocab.get(self.end_token) + + # ---- internal helpers for tag detection ---- + + def _has_start(self, token_ids: Sequence[int], text: str) -> bool: + """Check whether the start tag is present.""" + if self.start_token_id is not None: + return self.start_token_id in token_ids + return self.start_token in text + + def _has_end(self, token_ids: Sequence[int], text: str) -> bool: + """Check whether the end tag is present.""" + if self.end_token_id is not None: + return self.end_token_id in token_ids + return self.end_token in text + + def _is_single_start_token(self, delta_token_ids: Sequence[int], delta_text: str) -> bool: + """Check if the delta is exactly the start tag (single token).""" + if self.start_token_id is not None: + return len(delta_token_ids) == 1 and delta_token_ids[0] == self.start_token_id + return delta_text == self.start_token + + def _is_single_end_token(self, delta_token_ids: Sequence[int], delta_text: str) -> bool: + """Check if the delta is exactly the end tag (single token).""" + if self.end_token_id is not None: + return len(delta_token_ids) == 1 and delta_token_ids[0] == self.end_token_id + return delta_text == self.end_token + + def _split_at_end_token(self, text: str) -> tuple[str, str]: + """Split text at the end token, returning (before, after).""" + idx = text.find(self.end_token) + return text[:idx], text[idx + len(self.end_token):] + + # ---- public API ---- + + def extract_reasoning_content_streaming( + self, + delta_text: str, + delta_token_ids: Sequence[int], + request: object, + **kwargs, + ) -> DeltaMessage | None: + state = get_streaming_state(request) + previous_text = state.previous_text + previous_token_ids = state.previous_token_ids + + # Handle single special tokens + if self._is_single_end_token(delta_token_ids, delta_text): + return DeltaMessage(content='') + if self._is_single_start_token(delta_token_ids, delta_text): + return DeltaMessage(content='') + + # Check if start tag is in previous tokens + if self._has_start(previous_token_ids, previous_text): + if self._has_end(delta_token_ids, delta_text): + # start in previous, end in delta -> split at end tag + reasoning_content, content = self._split_at_end_token(delta_text) + return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) + elif self._has_end(previous_token_ids, previous_text): + # start in previous, end in previous -> reasoning is done + return DeltaMessage(content=delta_text) + else: + # start in previous, no end yet -> still reasoning + return DeltaMessage(reasoning_content=delta_text) + + # Check if start tag is in delta + if self._has_start(delta_token_ids, delta_text): + if self._has_end(delta_token_ids, delta_text): + # Both start and end in delta -> extract between them + start_idx = delta_text.find(self.start_token) + end_idx = delta_text.find(self.end_token) + reasoning_content = delta_text[start_idx + len(self.start_token):end_idx] + content = delta_text[end_idx + len(self.end_token):] + return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) + else: + # start in delta, no end -> reasoning begins + return DeltaMessage(reasoning_content=delta_text) + + # No start tag in previous or delta. + # Still need to check for end tag (model may omit start tag). + # Ref: https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f + if self._has_end(delta_token_ids, delta_text): + reasoning_content, content = self._split_at_end_token(delta_text) + return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) + elif self._has_end(previous_token_ids, previous_text): + # end in previous -> reasoning finished earlier + return DeltaMessage(content=delta_text) + else: + # no end anywhere -> still in reasoning + return DeltaMessage(reasoning_content=delta_text) + + def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, **kwargs) -> tuple[str, str]: + # If end tag is not present, behavior depends on on_missing_end_tag + if self.end_token not in model_output: + if self.on_missing_end_tag == 'reasoning': + return model_output, None + else: + return None, model_output + + # Add start tag if missing (compatibility with models that omit it) + if self.start_token not in model_output: + model_output = f'{self.start_token}{model_output}' + + # Extract reasoning content using str.find() + slicing + start_idx = model_output.find(self.start_token) + end_idx = model_output.find(self.end_token) + reasoning_content = model_output[start_idx + len(self.start_token):end_idx] + final_output = model_output[end_idx + len(self.end_token):] + + if self.strip_newlines: + reasoning_content = reasoning_content.strip('\n') + + return ( + reasoning_content if reasoning_content else None, + final_output if final_output else None, + ) diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_parser.py index e104511d76..b7e9676472 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_parser.py @@ -9,6 +9,7 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall) +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -41,14 +42,12 @@ def get_argments(self, obj): def extract_tool_calls_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> Union[DeltaMessage, None]: + state = get_streaming_state(request) + current_text = state.current_text if '<|action_start|>' not in current_text: self.position = len(current_text) return DeltaMessage(content=delta_text) diff --git a/lmdeploy/serve/openai/tool_parser/llama3_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_parser.py index 1c4eaf35d6..efc3118f38 100644 --- a/lmdeploy/serve/openai/tool_parser/llama3_parser.py +++ b/lmdeploy/serve/openai/tool_parser/llama3_parser.py @@ -9,6 +9,7 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall) +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -64,14 +65,12 @@ def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) def extract_tool_calls_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> Union[DeltaMessage, None]: + state = get_streaming_state(request) + current_text = state.current_text if not (current_text.startswith(self.bot_token) or current_text.startswith('{')): return DeltaMessage(content=delta_text) diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_parser.py index 9cd68b04e4..7e041c5915 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen2d5_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen2d5_parser.py @@ -9,6 +9,7 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall) +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -36,14 +37,12 @@ def get_argments(self, obj): def extract_tool_calls_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> Union[DeltaMessage, None]: + state = get_streaming_state(request) + current_text = state.current_text if self.tool_start_token not in current_text: self.position = len(current_text) return DeltaMessage(content=delta_text) diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_parser.py index f1a9635d6c..9389b25d00 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3_parser.py @@ -8,6 +8,7 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall) +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -112,11 +113,7 @@ def _parse_delta_tool_call(self, parser_state: ParserState, tool_content: str) - def extract_tool_calls_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> Union[DeltaMessage, None]: @@ -125,6 +122,9 @@ def extract_tool_calls_streaming( This method processes incremental model output to extract tool calls, reasoning content, and regular text content in a streaming fashion. It maintains parser state between calls to handle partial outputs. """ + state = get_streaming_state(request) + current_text = state.current_text + parser_state = getattr(request, '_tool_parser_state', None) if parser_state is None: parser_state = ParserState() diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py index 24ee53c7a8..3137c5db19 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py @@ -8,6 +8,7 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall) +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -119,14 +120,12 @@ def _extract_params(self, content: str) -> Tuple[Optional[str], Dict[str, Any], def extract_tool_calls_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> Union[DeltaMessage, None]: + state = get_streaming_state(request) + current_text = state.current_text parser_state = getattr(request, '_tool_parser_state', None) if parser_state is None: diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index 89ed8091ce..27330605e5 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -48,11 +48,7 @@ def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) def extract_tool_calls_streaming( self, - previous_text: str, - current_text: str, delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> Union[DeltaMessage, None]: @@ -60,8 +56,16 @@ def extract_tool_calls_streaming( from an incomplete response; for use when handling tool calls and streaming. - Has to be an instance method because it requires state - the current tokens/diffs, but also the information - about what has previously been parsed and extracted (see constructor) + Args: + delta_text: The new text chunk for this iteration. + delta_token_ids: The new token ids for this chunk. + request: The request object; a ``StreamingParserState`` is attached + to it via ``get_streaming_state(request)`` so that previous / + current text and token ids are available. + + Has to be an instance method because it requires state - the current + tokens/diffs, but also the information about what has previously been + parsed and extracted (see constructor). """ raise NotImplementedError('AbstractToolParser.extract_tool_calls_streaming has not been ' 'implemented!') From 15729004e93c74a29ed840491cc858299f8d5b1f Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Tue, 10 Mar 2026 09:27:03 +0000 Subject: [PATCH 02/14] rename file --- lmdeploy/serve/openai/reasoning_parser/__init__.py | 2 +- ...wq_reasoning_parser.py => qwen_reasoning_parser.py} | 0 lmdeploy/serve/openai/tool_parser/__init__.py | 10 +++++----- .../{internlm2_parser.py => internlm2_tool_parser.py} | 1 - .../{llama3_parser.py => llama3_tool_parser.py} | 0 .../{qwen2d5_parser.py => qwen2d5_tool_parser.py} | 0 .../{qwen3_parser.py => qwen3_tool_parser.py} | 0 ...{qwen3coder_parser.py => qwen3coder_tool_parser.py} | 2 +- tests/test_lmdeploy/test_qwen3_parser.py | 4 ++-- tests/test_lmdeploy/test_qwen3coder_parser.py | 2 +- 10 files changed, 10 insertions(+), 11 deletions(-) rename lmdeploy/serve/openai/reasoning_parser/{qwen_qwq_reasoning_parser.py => qwen_reasoning_parser.py} (100%) rename lmdeploy/serve/openai/tool_parser/{internlm2_parser.py => internlm2_tool_parser.py} (98%) rename lmdeploy/serve/openai/tool_parser/{llama3_parser.py => llama3_tool_parser.py} (100%) rename lmdeploy/serve/openai/tool_parser/{qwen2d5_parser.py => qwen2d5_tool_parser.py} (100%) rename lmdeploy/serve/openai/tool_parser/{qwen3_parser.py => qwen3_tool_parser.py} (100%) rename lmdeploy/serve/openai/tool_parser/{qwen3coder_parser.py => qwen3coder_tool_parser.py} (99%) diff --git a/lmdeploy/serve/openai/reasoning_parser/__init__.py b/lmdeploy/serve/openai/reasoning_parser/__init__.py index c396a8b3ed..e338f4b848 100644 --- a/lmdeploy/serve/openai/reasoning_parser/__init__.py +++ b/lmdeploy/serve/openai/reasoning_parser/__init__.py @@ -1,6 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from .qwen_qwq_reasoning_parser import QwenQwQReasoningParser +from .qwen_reasoning_parser import QwenQwQReasoningParser from .reasoning_parser import (ReasoningParser, ReasoningParserManager, StreamingParserState, ThinkingReasoningParser, get_streaming_state) diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_qwq_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py similarity index 100% rename from lmdeploy/serve/openai/reasoning_parser/qwen_qwq_reasoning_parser.py rename to lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py diff --git a/lmdeploy/serve/openai/tool_parser/__init__.py b/lmdeploy/serve/openai/tool_parser/__init__.py index e1e2b2726e..51446a9e16 100644 --- a/lmdeploy/serve/openai/tool_parser/__init__.py +++ b/lmdeploy/serve/openai/tool_parser/__init__.py @@ -1,9 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .internlm2_parser import Internlm2ToolParser -from .llama3_parser import Llama3JsonToolParser -from .qwen2d5_parser import Qwen2d5ToolParser -from .qwen3_parser import Qwen3ToolParser -from .qwen3coder_parser import Qwen3CoderToolParser +from .internlm2_tool_parser import Internlm2ToolParser +from .llama3_tool_parser import Llama3JsonToolParser +from .qwen2d5_tool_parser import Qwen2d5ToolParser +from .qwen3_tool_parser import Qwen3ToolParser +from .qwen3coder_tool_parser import Qwen3CoderToolParser from .tool_parser import ToolParser, ToolParserManager __all__ = [ diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py similarity index 98% rename from lmdeploy/serve/openai/tool_parser/internlm2_parser.py rename to lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index b7e9676472..ae1cc5471b 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -1,5 +1,4 @@ # Copyright (c) OpenMMLab. All rights reserved. -# modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/tool_parsers import json from typing import Dict, Sequence, Union diff --git a/lmdeploy/serve/openai/tool_parser/llama3_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py similarity index 100% rename from lmdeploy/serve/openai/tool_parser/llama3_parser.py rename to lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py similarity index 100% rename from lmdeploy/serve/openai/tool_parser/qwen2d5_parser.py rename to lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py similarity index 100% rename from lmdeploy/serve/openai/tool_parser/qwen3_parser.py rename to lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py similarity index 99% rename from lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py rename to lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index 3137c5db19..b4f7c70c67 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -168,7 +168,7 @@ def extract_tool_calls_streaming( if k not in parser_state.emitted_params: prefix = ', ' if len(parser_state.emitted_params) > 0 else '' serialized = json.dumps(v, ensure_ascii=False) - json_fragments.append(f'{prefix}"{k}": {serialized}') + json_fragments.append(f'{prefix}\"{k}\": {serialized}') parser_state.emitted_params.add(k) if is_func_closed and not getattr(parser_state, 'json_closed', False): diff --git a/tests/test_lmdeploy/test_qwen3_parser.py b/tests/test_lmdeploy/test_qwen3_parser.py index 3a837d73a3..2354d8b7e2 100644 --- a/tests/test_lmdeploy/test_qwen3_parser.py +++ b/tests/test_lmdeploy/test_qwen3_parser.py @@ -10,8 +10,8 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, DeltaMessage, DeltaToolCall, UsageInfo) -from lmdeploy.serve.openai.reasoning_parser.qwen_qwq_reasoning_parser import QwenQwQReasoningParser -from lmdeploy.serve.openai.tool_parser.qwen3_parser import Qwen3ToolParser +from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenQwQReasoningParser +from lmdeploy.serve.openai.tool_parser.qwen3_tool_parser import Qwen3ToolParser TestExpects = collections.namedtuple('TestExpects', 'func_name location') diff --git a/tests/test_lmdeploy/test_qwen3coder_parser.py b/tests/test_lmdeploy/test_qwen3coder_parser.py index b84735a40c..80d4c446e8 100644 --- a/tests/test_lmdeploy/test_qwen3coder_parser.py +++ b/tests/test_lmdeploy/test_qwen3coder_parser.py @@ -10,7 +10,7 @@ from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, ChatCompletionResponse, ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, ChatCompletionStreamResponse, ChatMessage, DeltaMessage, DeltaToolCall, UsageInfo) -from lmdeploy.serve.openai.tool_parser.qwen3coder_parser import Qwen3CoderToolParser +from lmdeploy.serve.openai.tool_parser.qwen3coder_tool_parser import Qwen3CoderToolParser TestExpects = collections.namedtuple('TestExpects', 'func_name kwargs') From b895d53c0fb9414afa668410a2d0aa3df60b5541 Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Tue, 10 Mar 2026 09:40:19 +0000 Subject: [PATCH 03/14] minor fix --- lmdeploy/serve/openai/api_server.py | 7 ++--- .../reasoning_parser/reasoning_parser.py | 29 ++++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index e0994a2e26..6b7b67a52d 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -509,8 +509,7 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: has_parser = VariableInterface.tool_parser is not None or VariableInterface.reasoning_parser is not None streaming_tools = False # Shared state for streaming parsers (previous/current text & token ids) - if has_parser: - parser_state = get_streaming_state(request) + parser_state = get_streaming_state(request) if has_parser else None async for res in result_generator: logprobs, usage = None, None if gen_logprobs and res.logprobs: @@ -532,7 +531,7 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: res.finish_reason = 'tool_calls' else: delta_message = DeltaMessage(role='assistant', content=res.response) - if has_parser: + if parser_state is not None: parser_state.update(res.response, delta_token_ids) if request.tool_choice != 'none' and VariableInterface.tool_parser is not None: if res.finish_reason == 'stop' and streaming_tools is True: @@ -553,7 +552,7 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: if reasoning_delta is not None: delta_message.reasoning_content = reasoning_delta.reasoning_content delta_message.content = reasoning_delta.content - if has_parser: + if parser_state is not None: parser_state.step() if request.return_token_ids: delta_message.gen_tokens = delta_token_ids diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index 9a6c5d90d1..63218a33db 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -1,8 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/reasoning_parsers -from dataclasses import dataclass, field +from dataclasses import dataclass from functools import cached_property -from typing import Sequence from mmengine import Registry @@ -20,18 +19,18 @@ class StreamingParserState: """ previous_text: str = '' current_text: str = '' - previous_token_ids: list[int] = field(default_factory=list) - current_token_ids: list[int] = field(default_factory=list) + previous_token_ids: list[int] = [] + current_token_ids: list[int] = [] - def update(self, delta_text: str, delta_token_ids: Sequence[int]) -> None: + def update(self, delta_text: str, delta_token_ids: list[int]) -> None: """Accumulate new delta into current_text / current_token_ids.""" self.current_text += delta_text - self.current_token_ids = self.current_token_ids + list(delta_token_ids) + self.current_token_ids.extend(delta_token_ids) def step(self) -> None: """Advance: copy current -> previous (call at end of each iteration).""" self.previous_text = self.current_text - self.previous_token_ids = list(self.current_token_ids) + self.previous_token_ids = self.current_token_ids def get_streaming_state(request: object) -> StreamingParserState: @@ -58,7 +57,7 @@ def vocab(self) -> dict[str, int]: def extract_reasoning_content_streaming( self, delta_text: str, - delta_token_ids: Sequence[int], + delta_token_ids: list[int], request: object, **kwargs, ) -> DeltaMessage | None: @@ -136,27 +135,25 @@ def __init__(self, tokenizer: object): self.start_token_id: int = self.vocab.get(self.start_token) self.end_token_id: int = self.vocab.get(self.end_token) - # ---- internal helpers for tag detection ---- - - def _has_start(self, token_ids: Sequence[int], text: str) -> bool: + def _has_start(self, token_ids: list[int], text: str) -> bool: """Check whether the start tag is present.""" if self.start_token_id is not None: return self.start_token_id in token_ids return self.start_token in text - def _has_end(self, token_ids: Sequence[int], text: str) -> bool: + def _has_end(self, token_ids: list[int], text: str) -> bool: """Check whether the end tag is present.""" if self.end_token_id is not None: return self.end_token_id in token_ids return self.end_token in text - def _is_single_start_token(self, delta_token_ids: Sequence[int], delta_text: str) -> bool: + def _is_single_start_token(self, delta_token_ids: list[int], delta_text: str) -> bool: """Check if the delta is exactly the start tag (single token).""" if self.start_token_id is not None: return len(delta_token_ids) == 1 and delta_token_ids[0] == self.start_token_id return delta_text == self.start_token - def _is_single_end_token(self, delta_token_ids: Sequence[int], delta_text: str) -> bool: + def _is_single_end_token(self, delta_token_ids: list[int], delta_text: str) -> bool: """Check if the delta is exactly the end tag (single token).""" if self.end_token_id is not None: return len(delta_token_ids) == 1 and delta_token_ids[0] == self.end_token_id @@ -167,12 +164,10 @@ def _split_at_end_token(self, text: str) -> tuple[str, str]: idx = text.find(self.end_token) return text[:idx], text[idx + len(self.end_token):] - # ---- public API ---- - def extract_reasoning_content_streaming( self, delta_text: str, - delta_token_ids: Sequence[int], + delta_token_ids: list[int], request: object, **kwargs, ) -> DeltaMessage | None: From 35b404cd10cbfaba0b25e04b19999ae2ea045ebe Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Thu, 26 Mar 2026 14:02:08 +0000 Subject: [PATCH 04/14] refactor --- lmdeploy/serve/openai/api_server.py | 68 ++--- .../serve/openai/reasoning_parser/__init__.py | 17 +- .../deepseek_r1_reasoning_parser.py | 26 +- .../deepseek_v3_reasoning_parser.py | 49 ++++ .../identity_reasoning_parser.py | 39 +++ .../reasoning_parser/qwen_reasoning_parser.py | 61 +++- .../reasoning_parser/reasoning_parser.py | 184 +++++------- .../test_qwen_reasoning_parser.py | 264 ++++++++++++++++++ tests/test_lmdeploy/test_qwen3_parser.py | 12 +- tests/test_lmdeploy/test_qwen3coder_parser.py | 6 +- 10 files changed, 539 insertions(+), 187 deletions(-) create mode 100644 lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py create mode 100644 lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py create mode 100644 tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 015092098a..b750813d1c 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -97,9 +97,9 @@ class VariableInterface: proxy_url: str | None = None api_server_url: str | None = None # following are for reasoning parsers - reasoning_parser: ReasoningParser | None = None + reasoning_parser_cls: type[ReasoningParser] | None = None # following is for tool parsers - tool_parser: ToolParser | None = None + tool_parser_cls: type[ToolParser] | None = None allow_terminate_by_client: bool = False enable_abort_handling: bool = False @@ -542,16 +542,21 @@ def create_stream_response_json(index: int, return response_json + tokenizer = VariableInterface.async_engine.tokenizer + reasoning_parser, tool_parser = None, None + if VariableInterface.reasoning_parser_cls is not None: + reasoning_parser = VariableInterface.reasoning_parser_cls(tokenizer, **chat_template_kwargs) + if VariableInterface.tool_parser_cls is not None: + tool_parser = VariableInterface.tool_parser_cls(tokenizer, **chat_template_kwargs) + async def completion_stream_generator() -> AsyncGenerator[str, None]: - has_parser = VariableInterface.tool_parser is not None or VariableInterface.reasoning_parser is not None streaming_tools = False # Shared state for streaming parsers (previous/current text & token ids) - parser_state = get_streaming_state(request) if has_parser else None + parser_state = get_streaming_state(request) if reasoning_parser or tool_parser else None async for res in result_generator: logprobs, usage = None, None if gen_logprobs and res.logprobs: - logprobs = _create_chat_completion_logprobs(VariableInterface.async_engine.tokenizer, res.token_ids, - res.logprobs) + logprobs = _create_chat_completion_logprobs(tokenizer, res.token_ids, res.logprobs) # Only stream chunk `usage` in the final chunk according to OpenAI API spec if (res.finish_reason and request.stream_options and request.stream_options.include_usage): total_tokens = sum([res.input_token_len, res.generate_token_len]) @@ -570,10 +575,10 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: delta_message = DeltaMessage(role='assistant', content=res.response) if parser_state is not None: parser_state.update(res.response, delta_token_ids) - if request.tool_choice != 'none' and VariableInterface.tool_parser is not None: + if request.tool_choice != 'none' and tool_parser is not None: if res.finish_reason == 'stop' and streaming_tools is True: res.finish_reason = 'tool_calls' - tool_delta = VariableInterface.tool_parser.extract_tool_calls_streaming( + tool_delta = tool_parser.extract_tool_calls_streaming( delta_text=delta_message.content, delta_token_ids=delta_token_ids, request=request) if tool_delta is not None: delta_message.tool_calls = tool_delta.tool_calls @@ -581,10 +586,10 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: if isinstance(tool_delta.tool_calls, list) and len(tool_delta.tool_calls): streaming_tools = True elif (request.tool_choice != 'none' and request.tools is not None - and VariableInterface.tool_parser is None): + and tool_parser is None): logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') - if VariableInterface.reasoning_parser is not None and enable_thinking is not False: - reasoning_delta = VariableInterface.reasoning_parser.extract_reasoning_content_streaming( + if reasoning_parser and enable_thinking is not False: + reasoning_delta = reasoning_parser.extract_reasoning_streaming( delta_text=delta_message.content or '', delta_token_ids=delta_token_ids, request=request) if reasoning_delta is not None: delta_message.reasoning_content = reasoning_delta.reasoning_content @@ -636,9 +641,9 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: else: tool_calls = None reasoning_content = None - if request.tool_choice != 'none' and VariableInterface.tool_parser is not None: + if request.tool_choice != 'none' and tool_parser is not None: try: - tool_call_info = VariableInterface.tool_parser.extract_tool_calls(text, request=request) + tool_call_info = tool_parser.extract_tool_calls(text, request=request) text, tool_calls = tool_call_info.content, tool_call_info.tool_calls if isinstance(tool_calls, list) and len(tool_calls): if final_res.finish_reason == 'stop': @@ -647,11 +652,11 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: except Exception as e: logger.error(f'Failed to parse {text}. Exception: {e}.') return create_error_response(HTTPStatus.BAD_REQUEST, 'Failed to parse fc related info to json format!') - elif request.tool_choice != 'none' and request.tools is not None and VariableInterface.tool_parser is None: + elif request.tool_choice != 'none' and request.tools is not None and tool_parser is None: logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') - if VariableInterface.reasoning_parser is not None and enable_thinking is not False: - reasoning_content, text = VariableInterface.reasoning_parser.extract_reasoning_content(text, request) + if reasoning_parser and enable_thinking is not False: + reasoning_content, text = reasoning_parser.extract_reasoning(text, request) message = ChatMessage(role='assistant', content=text, @@ -1314,26 +1319,21 @@ async def dispatch(self, request: Request, call_next): return response -def set_parsers(reasoning_parser: str | None = None, tool_parser: str | None = None): +def set_parsers(reasoning_parser_name: str | None = None, tool_parser_name: str | None = None, **kwargs): """Set tool parser and reasoning parsers.""" - # set reasoning parser - if reasoning_parser is not None: - if reasoning_parser in ReasoningParserManager.module_dict: - tokenizer = VariableInterface.async_engine.tokenizer - VariableInterface.reasoning_parser = ReasoningParserManager.get(reasoning_parser)(tokenizer) + if reasoning_parser_name is not None: + if reasoning_parser_name in ReasoningParserManager.module_dict: + VariableInterface.reasoning_parser_cls = ReasoningParserManager.get(reasoning_parser_name) else: - raise ValueError( - f'The reasoning parser {reasoning_parser} is not in the parser list: {ReasoningParserManager.module_dict.keys()}' # noqa - ) - # set tool parsers - if tool_parser is not None: - if tool_parser in ToolParserManager.module_dict: - tokenizer = VariableInterface.async_engine.tokenizer - VariableInterface.tool_parser = ToolParserManager.get(tool_parser)(tokenizer) + raise ValueError(f'The reasoning parser {reasoning_parser_name} is not in the parser list: ' + f'{ReasoningParserManager.module_dict.keys()}') + + if tool_parser_name is not None: + if tool_parser_name in ToolParserManager.module_dict: + VariableInterface.tool_parser_cls = ToolParserManager.get(tool_parser_name) else: - raise ValueError( - f'The reasoning parser {tool_parser} is not in the parser list: {ToolParserManager.module_dict.keys()}' # noqa - ) + raise ValueError(f'The tool parser {tool_parser_name} is not in the parser list: ' + f'{ToolParserManager.module_dict.keys()}') def mount_metrics(app: FastAPI, backend_config: PytorchEngineConfig | TurbomindEngineConfig): @@ -1452,7 +1452,7 @@ def serve(model_path: str, being printed in log. Default: Unlimited max_concurrent_requests: This refers to the number of concurrent requests that the server can handle. The server is designed to - process the engine’s tasks once the maximum number of concurrent + process the engine's tasks once the maximum number of concurrent requests is reached, regardless of any additional requests sent by clients concurrently during that time. Default to None. reasoning_parser (str): The reasoning parser name. diff --git a/lmdeploy/serve/openai/reasoning_parser/__init__.py b/lmdeploy/serve/openai/reasoning_parser/__init__.py index e338f4b848..b26208ba2a 100644 --- a/lmdeploy/serve/openai/reasoning_parser/__init__.py +++ b/lmdeploy/serve/openai/reasoning_parser/__init__.py @@ -1,8 +1,15 @@ # Copyright (c) OpenMMLab. All rights reserved. from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from .qwen_reasoning_parser import QwenQwQReasoningParser -from .reasoning_parser import (ReasoningParser, ReasoningParserManager, StreamingParserState, ThinkingReasoningParser, - get_streaming_state) +from .deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser +from .identity_reasoning_parser import IdentityReasoningParser +from .qwen_reasoning_parser import QwenReasoningParser +from .reasoning_parser import ( + ReasoningParser, + ReasoningParserManager, + StreamingParserState, + ThinkingReasoningParser, + get_streaming_state, +) __all__ = [ 'ReasoningParser', @@ -11,5 +18,7 @@ 'ThinkingReasoningParser', 'get_streaming_state', 'DeepSeekR1ReasoningParser', - 'QwenQwQReasoningParser', + 'QwenReasoningParser', + 'IdentityReasoningParser', + 'DeepSeekV3ReasoningParser', ] diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py index ca9dbaa67e..b81e9da8cf 100644 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py @@ -1,25 +1,15 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser +from .qwen_reasoning_parser import QwenReasoningParser +from .reasoning_parser import ReasoningParserManager @ReasoningParserManager.register_module(name='deepseek-r1') -class DeepSeekR1ReasoningParser(ThinkingReasoningParser): +class DeepSeekR1ReasoningParser(QwenReasoningParser): """Reasoning parser for DeepSeek R1 model. - Uses ... tokens. When the end tag is missing in - non-streaming mode, the entire output is treated as reasoning content - (DeepSeek R1 may omit the start tag). - - Ref: https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f + DeepSeek R1 always put tag to user's prompt. see more details in + https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f + Since DeepSeek-R1 and Qwen3-Thinking models have the same reasoning behavior, + we remove its original implementation and directly use QwenReasoningParser. """ - - start_token = '' - end_token = '' - strip_newlines = False - on_missing_end_tag = 'reasoning' - - def __init__(self, tokenizer: object): - super().__init__(tokenizer) - if self.start_token_id is None or self.end_token_id is None: - raise RuntimeError('DeepSeek R1 reasoning parser could not locate ' - 'think start/end tokens in the tokenizer!') + pass diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py new file mode 100644 index 0000000000..eecb96d8d6 --- /dev/null +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py @@ -0,0 +1,49 @@ +# Copyright (c) OpenMMLab. All rights reserved. + +from typing import TYPE_CHECKING + +from lmdeploy.serve.openai.protocol import DeltaMessage + +from .identity_reasoning_parser import IdentityReasoningParser +from .reasoning_parser import ReasoningParser + +if TYPE_CHECKING: + from lmdeploy.serve.openai.protocol import ChatCompletionRequest + +class DeepSeekV3ReasoningParser(ReasoningParser): + """The reasoning behavior of the DeepSeek V3.1 model varies depending on + the `enable_thinking` parameter. + + When set to True, a tag is added to the user's prompt, which corresponds to the thinking mode + of DeepSeek R1. + When `enable_thinking` is None, the thinking mode is disabled. In this case, the parser falls back to + the identity parser, which treats the entire model output as content and ignores any reasoning. + """ + + def __init__(self, tokenizer: object, **kwargs): + super().__init__(tokenizer, **kwargs) + + enable_thinking = bool(kwargs.get('enable_thinking', False)) + self._parser: ReasoningParser + if enable_thinking: + from .qwen_reasoning_parser import QwenReasoningParser as DeepSeekR1ReasoningParser + self._parser = DeepSeekR1ReasoningParser(tokenizer, **kwargs) + else: + self._parser = IdentityReasoningParser(tokenizer, **kwargs) + + def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest') -> tuple[str | None, str | None]: + return self._parser.extract_reasoning(model_output, request) + + def extract_reasoning_streaming( + self, + delta_text: str, + delta_token_ids: list[int], + request: object, + **kwargs, + ) -> DeltaMessage | None: + return self._parser.extract_reasoning_streaming( + delta_text, + delta_token_ids, + request, + **kwargs, + ) diff --git a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py new file mode 100644 index 0000000000..f0c818327c --- /dev/null +++ b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py @@ -0,0 +1,39 @@ +# Copyright (c) OpenMMLab. All rights reserved. + +# modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/identity_reasoning_parser.py +from typing import TYPE_CHECKING + +from lmdeploy.serve.openai.protocol import DeltaMessage +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser + +if TYPE_CHECKING: + from lmdeploy.serve.openai.protocol import ChatCompletionRequest + + +class IdentityReasoningParser(ReasoningParser): + """Identity reasoning parser. + + This parser does not attempt to parse or strip out reasoning tokens. It treats the entire model output as content + and ignores reasoning. + """ + + def __init__(self, tokenizer, **kwargs): + super().__init__(tokenizer, **kwargs) + + + def extract_reasoning_streaming( + self, + delta_text: str, + delta_token_ids: list[int], + request: object, + **kwargs, + ) -> DeltaMessage | None: + # Just wrap delta_text as content, ignore reasoning + if delta_text: + return DeltaMessage(content=delta_text) + return None + + def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest') -> tuple[str | None, str | None]: + # No reasoning separation: return None for reasoning, + # and full model_output as content + return None, model_output diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py index 82866ad52c..bf041de428 100644 --- a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py @@ -1,18 +1,59 @@ # Copyright (c) OpenMMLab. All rights reserved. -from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser +# modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/qwen3_reasoning_parser.py +from lmdeploy.serve.openai.protocol import DeltaMessage -@ReasoningParserManager.register_module(name=['qwen-qwq', 'intern-s1']) -class QwenQwQReasoningParser(ThinkingReasoningParser): - """Reasoning parser for Qwen QwQ / Qwen3 / InternLM-S1 models. +from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser, get_streaming_state - Uses ... tokens. When the end tag is missing in - non-streaming mode, the entire output is treated as normal content - (not reasoning). Leading/trailing newlines in reasoning content are - stripped. + +@ReasoningParserManager.register_module(name=['qwen-qwq', 'qwen3', 'intern-s1', 'deepseeek-r1']) +class QwenReasoningParser(ThinkingReasoningParser): + """Reasoning parser for Qwen QwQ / Qwen3 / Intern-S / Qwen3.5 models. + + Qwen3 models, such as Qwen3-8B, Qwen3-**-Instruct, generate tag if enable_thinking is True. + However, Qwen3-Thinking models and Qwen3.5 models put in user's prompt, thus they don't + generate tag. Intern-S models hold the same behavior as Qwen3-Thinking models. + + This parser handles both styles: if appears in the generated output + it is stripped before extraction (non-streaming) or skipped (streaming). """ start_token = '' end_token = '' - strip_newlines = True - on_missing_end_tag = 'content' + + def extract_reasoning_streaming(self, delta_text: str, delta_token_ids: list[int], + request: object, **kwargs) -> DeltaMessage | None: + state = get_streaming_state(request) + previous_token_ids = state.previous_token_ids + + # Strip from delta if present (old template / edge case where the model generates itself). + if self.start_token_id in delta_token_ids: + start_idx = delta_text.find(self.start_token) + if start_idx >= 0: + delta_text = delta_text[start_idx + len(self.start_token) :] + + if self.end_token_id in delta_token_ids: + # End token in this delta: split reasoning from content. + end_index = delta_text.find(self.end_token) + if end_index >= 0: + reasoning = delta_text[:end_index] + content = delta_text[end_index + len(self.end_token) :] + if not reasoning and not content: + return None + return DeltaMessage( + reasoning_content=reasoning if reasoning else None, + content=content if content else None, + ) + # end_token_id in IDs but not in text (already stripped) + return None + + # No end token in this delta. + if not delta_text: + # Nothing left after stripping start token. + return None + elif self.end_token_id in previous_token_ids: + # End token already passed: everything is content now. + return DeltaMessage(content=delta_text) + else: + # No end token yet: still in reasoning phase. + return DeltaMessage(reasoning_content=delta_text) diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index 5e9900dcdf..7de8cf71a6 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -1,6 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/reasoning_parsers -from dataclasses import dataclass +from dataclasses import dataclass, field from functools import cached_property from mmengine import Registry @@ -19,8 +19,8 @@ class StreamingParserState: """ previous_text: str = '' current_text: str = '' - previous_token_ids: list[int] = [] - current_token_ids: list[int] = [] + previous_token_ids: list[int] = field(default_factory=list) + current_token_ids: list[int] = field(default_factory=list) def update(self, delta_text: str, delta_token_ids: list[int]) -> None: """Accumulate new delta into current_text / current_token_ids.""" @@ -45,7 +45,7 @@ def get_streaming_state(request: object) -> StreamingParserState: class ReasoningParser: """Abstract base class for reasoning content parsers.""" - def __init__(self, tokenizer: object): + def __init__(self, tokenizer: object, **kwargs): self.model_tokenizer = tokenizer @cached_property @@ -54,7 +54,7 @@ def vocab(self) -> dict[str, int]: # whereas all tokenizers have .get_vocab() return self.model_tokenizer.get_vocab() - def extract_reasoning_content_streaming( + def extract_reasoning_streaming( self, delta_text: str, delta_token_ids: list[int], @@ -76,10 +76,10 @@ def extract_reasoning_content_streaming( Returns a DeltaMessage with reasoning_content and/or content fields, or None if the delta should be skipped. """ - raise NotImplementedError('ReasoningParser.extract_reasoning_content_streaming ' + raise NotImplementedError('ReasoningParser.extract_reasoning_streaming ' 'has not been implemented!') - def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, + def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', **kwargs) -> tuple[str | None, str | None]: """Extract reasoning content from a complete model-generated string. @@ -93,15 +93,14 @@ def extract_reasoning_content(self, model_output: str, request: ChatCompletionRe Returns: A tuple of (reasoning_content, final_output). Either may be None. """ - raise NotImplementedError('ReasoningParser.extract_reasoning_content ' + raise NotImplementedError('ReasoningParser.extract_reasoning ' 'has not been implemented!') class ThinkingReasoningParser(ReasoningParser): """Base class for reasoning parsers that use ... style tags. - Subclasses only need to set `start_token`, `end_token`, and optionally - override `strip_newlines` and `on_missing_start_tag` to customize behavior. + Subclasses only need to set `start_token`, `end_token`. This parser uses a two-step detection strategy (inspired by vllm): 1. First check token_ids (fast integer comparison) to determine whether @@ -112,25 +111,12 @@ class ThinkingReasoningParser(ReasoningParser): it falls back to string-based detection automatically. """ - # Subclasses should set these start_token: str = '' end_token: str = '' - # Whether to strip leading/trailing newlines from reasoning content - # in non-streaming extraction. - strip_newlines: bool = False - # Behavior when end_token is not found in non-streaming extraction: - # 'reasoning' -> treat entire output as reasoning (DeepSeek R1 behavior) - # 'content' -> treat entire output as content (QwQ/Qwen3 behavior) - on_missing_end_tag: str = 'content' - - def __init__(self, tokenizer: object): - super().__init__(tokenizer) - - if not self.model_tokenizer: - raise ValueError('The model tokenizer must be passed to the ' - 'ReasoningParser constructor during construction.') + def __init__(self, tokenizer: object, **kwargs): + super().__init__(tokenizer, **kwargs) # Try to resolve single token ids for fast detection. # If the tokenizer doesn't have them as single tokens, fall back to @@ -138,113 +124,87 @@ def __init__(self, tokenizer: object): self.start_token_id: int = self.vocab.get(self.start_token) self.end_token_id: int = self.vocab.get(self.end_token) - def _has_start(self, token_ids: list[int], text: str) -> bool: - """Check whether the start tag is present.""" - if self.start_token_id is not None: - return self.start_token_id in token_ids - return self.start_token in text - - def _has_end(self, token_ids: list[int], text: str) -> bool: - """Check whether the end tag is present.""" - if self.end_token_id is not None: - return self.end_token_id in token_ids - return self.end_token in text - - def _is_single_start_token(self, delta_token_ids: list[int], delta_text: str) -> bool: - """Check if the delta is exactly the start tag (single token).""" - if self.start_token_id is not None: - return len(delta_token_ids) == 1 and delta_token_ids[0] == self.start_token_id - return delta_text == self.start_token - - def _is_single_end_token(self, delta_token_ids: list[int], delta_text: str) -> bool: - """Check if the delta is exactly the end tag (single token).""" - if self.end_token_id is not None: - return len(delta_token_ids) == 1 and delta_token_ids[0] == self.end_token_id - return delta_text == self.end_token - - def _split_at_end_token(self, text: str) -> tuple[str, str]: - """Split text at the end token, returning (before, after).""" - idx = text.find(self.end_token) - return text[:idx], text[idx + len(self.end_token):] - - def extract_reasoning_content_streaming( + def extract_reasoning_streaming( self, delta_text: str, delta_token_ids: list[int], request: object, **kwargs, ) -> DeltaMessage | None: + """Extract reasoning content from a streaming model-generated string. + + Args: + delta_text: The new text chunk (may have been modified by the tool + parser before being passed here). + delta_token_ids: The new token ids for this chunk. + request: The request object; a ``StreamingParserState`` is attached + to it via ``get_streaming_state(request)`` so that previous / + current text and token ids are available. + + Returns a DeltaMessage with reasoning_content and/or content fields, + or None if the delta should be skipped. + """ state = get_streaming_state(request) - previous_text = state.previous_text previous_token_ids = state.previous_token_ids # Handle single special tokens - if self._is_single_end_token(delta_token_ids, delta_text): - return DeltaMessage(content='') - if self._is_single_start_token(delta_token_ids, delta_text): - return DeltaMessage(content='') + if len(delta_token_ids) == 1 and (delta_token_ids[0] in [self.start_token_id, self.end_token_id]): + return None # Check if start tag is in previous tokens - if self._has_start(previous_token_ids, previous_text): - if self._has_end(delta_token_ids, delta_text): - # start in previous, end in delta -> split at end tag - reasoning_content, content = self._split_at_end_token(delta_text) - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self._has_end(previous_token_ids, previous_text): - # start in previous, end in previous -> reasoning is done - return DeltaMessage(content=delta_text) - else: - # start in previous, no end yet -> still reasoning - return DeltaMessage(reasoning_content=delta_text) - - # Check if start tag is in delta - if self._has_start(delta_token_ids, delta_text): - if self._has_end(delta_token_ids, delta_text): + if self.start_token_id in previous_token_ids: + if self.end_token_id in delta_token_ids: # Both start and end in delta -> extract between them - start_idx = delta_text.find(self.start_token) end_idx = delta_text.find(self.end_token) - reasoning_content = delta_text[start_idx + len(self.start_token):end_idx] + reasoning_content = delta_text[:end_idx] content = delta_text[end_idx + len(self.end_token):] return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) + elif self.end_token_id in previous_token_ids: + # end in previous, no start -> reasoning is done + return DeltaMessage(content=delta_text) else: - # start in delta, no end -> reasoning begins + # start in previous, no end -> reasoning continues return DeltaMessage(reasoning_content=delta_text) - - # No start tag in previous or delta. - # Still need to check for end tag (model may omit start tag). - # Ref: https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - if self._has_end(delta_token_ids, delta_text): - reasoning_content, content = self._split_at_end_token(delta_text) - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self._has_end(previous_token_ids, previous_text): - # end in previous -> reasoning finished earlier - return DeltaMessage(content=delta_text) - else: - # no end anywhere -> still in reasoning - return DeltaMessage(reasoning_content=delta_text) - - def extract_reasoning_content(self, model_output: str, request: ChatCompletionRequest, **kwargs) -> tuple[str, str]: - # If end tag is not present, behavior depends on on_missing_end_tag - if self.end_token not in model_output: - if self.on_missing_end_tag == 'reasoning': - return model_output, None + elif self.start_token_id in delta_token_ids: + start_index = delta_text.find(self.start_token) + if self.end_token_id in delta_token_ids: + # Both start and end in delta -> extract between them + end_index = delta_text.find(self.end_token) + reasoning_content = delta_text[start_index + len(self.start_token) : end_index] + content = delta_text[end_index + len(self.end_token) :] + return DeltaMessage( + reasoning_content=reasoning_content, content=content if content else None + ) else: - return None, model_output - - # Add start tag if missing (compatibility with models that omit it) - if self.start_token not in model_output: - model_output = f'{self.start_token}{model_output}' + # start token in delta, no end token in delta, reasoning content continues + return DeltaMessage(reasoning_content=delta_text[start_index + len(self.start_token):]) + else: + # not find thinking start token + return DeltaMessage(content=delta_text) - # Extract reasoning content using str.find() + slicing - start_idx = model_output.find(self.start_token) - end_idx = model_output.find(self.end_token) - reasoning_content = model_output[start_idx + len(self.start_token):end_idx] - final_output = model_output[end_idx + len(self.end_token):] + def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', **kwargs) -> tuple[str, str]: + """Extract reasoning content from a complete model-generated string. - if self.strip_newlines: - reasoning_content = reasoning_content.strip('\n') + Args: + model_output: The model-generated string to extract reasoning content from. + request: The request object that was used to generate the model_output. - return ( - reasoning_content if reasoning_content else None, - final_output if final_output else None, + Returns: + A tuple of (reasoning_content, final_output). Either may be None. + """ + # Check if the start token is present in the model output, remove it + # if it is present. + model_output_parts = model_output.partition(self.start_token) + model_output = ( + model_output_parts[2] if model_output_parts[1] else model_output_parts[0] ) + + # For models that may not generate start token, + # assume the reasoning content is always at the start. + if self.end_token not in model_output: + return model_output, None + else: + reasoning, _, content = model_output.partition(self.end_token) + # If generation stops right after end-of-think, return null content + final_content = content or None + return reasoning, final_content diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py new file mode 100644 index 0000000000..5c101a683d --- /dev/null +++ b/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py @@ -0,0 +1,264 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Tests for QwenReasoningParser covering three model behavior modes. + +Scenario A – Thinking mode (Qwen3-8B, enable_thinking=True): + Model generates ``reasoning\\n\\nAnswer``. + +Scenario B – Non-thinking mode (Qwen3-8B, enable_thinking=False): + Model generates plain content with no ```` tags at all. + +Scenario C – Forceful Thinking (Qwen3-4B-Thinking-2507): + ```` is injected into the prompt by the chat template, so the + model's output starts directly with reasoning, then ````, then + the answer. No ```` appears in the generated output. +""" + +from __future__ import annotations + +import pytest + +from lmdeploy.serve.openai.protocol import ChatCompletionRequest +from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager, get_streaming_state +from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer + +# We use Qwen3-8B's tokenizer to simulate all the test cases. +MODEL_ID = 'Qwen/Qwen3-8B' + +@pytest.fixture(scope='module') +def tokenizer(): + try: + return HuggingFaceTokenizer(MODEL_ID) + except Exception as exc: # noqa: BLE001 + pytest.skip(f'Could not load tokenizer for {MODEL_ID}: {exc}') + + +@pytest.fixture() +def parser(tokenizer): + return QwenReasoningParser(tokenizer) + + +def simulate_pipeline_chunks( + tokenizer: HuggingFaceTokenizer, + full_text: str, + *, + chunk_size: int = 1, + skip_special_tokens: bool = True, + spaces_between_special_tokens: bool = True, +) -> list[tuple[str, list[int]]]: + """Split *full_text* into (delta_text, delta_token_ids) like + ``AsyncEngine.generate``.""" + all_ids = tokenizer.encode(full_text, add_bos=False, add_special_tokens=False) + state = DetokenizeState(0) + accumulated: list[int] = [] + chunks: list[tuple[str, list[int]]] = [] + offset = 0 + while offset < len(all_ids): + accumulated.extend(all_ids[offset:offset + chunk_size]) + offset += chunk_size + ids_offset_before = state.ids_offset + delta_text, state = tokenizer.detokenize_incrementally( + accumulated, + state, + skip_special_tokens=skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + ) + delta_ids = accumulated[ids_offset_before:len(accumulated)] + chunks.append((delta_text, delta_ids)) + return chunks + + +def run_reasoning_stream( + parser: QwenReasoningParser, + request: object, + chunks: list[tuple[str, list[int]]], +) -> tuple[str, str]: + """Mirror ``api_server`` ``completion_stream_generator`` parser loop. + + Returns (accumulated_reasoning, accumulated_content). + """ + state = get_streaming_state(request) + reasoning_acc = '' + content_acc = '' + for delta_text, delta_ids in chunks: + state.update(delta_text, delta_ids) + delta_msg = parser.extract_reasoning_streaming( + delta_text=delta_text or '', + delta_token_ids=delta_ids, + request=request, + ) + if delta_msg is not None: + if delta_msg.reasoning_content: + reasoning_acc += delta_msg.reasoning_content + if delta_msg.content is not None: + content_acc += delta_msg.content + state.step() + return reasoning_acc, content_acc + + +def _make_request(stream: bool = False) -> ChatCompletionRequest: + return ChatCompletionRequest(model=MODEL_ID, messages=[], stream=stream) + + +class TestExtractReasoning: + """Non-streaming ``extract_reasoning`` tests.""" + + def test_thinking_mode(self, parser): + """Qwen3-8B enable_thinking=True: + + ..reasoning..answer. + """ + full = '\nBrief chain of thought.\n\n\nThe answer is 42.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning == '\nBrief chain of thought.\n' + assert content == '\n\nThe answer is 42.' + + def test_non_thinking_mode(self, parser): + """Qwen3-8B enable_thinking=False: plain content, no tags.""" + full = 'The answer is 42.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning is None + assert content == 'The answer is 42.' + + def test_forceful_thinking(self, parser): + """Qwen3-4B-Thinking-2507: no in output, model starts with reasoning.""" + full = '\nBrief chain of thought.\n\n\nThe answer is 42.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning == '\nBrief chain of thought.\n' + assert content == '\n\nThe answer is 42.' + + def test_empty_reasoning(self, parser): + """Edge case: with empty reasoning body.""" + full = '\n\nThe answer is 42.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning is None + assert content == '\n\nThe answer is 42.' + + def test_only_reasoning_no_answer(self, parser): + """Edge case: reasoning present but no content after .""" + full = 'reasoning only' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning == 'reasoning only' + assert content is None + + def test_multiline_reasoning(self, parser): + """Longer, multi-line reasoning body.""" + reasoning_text = ( + '\nStep 1: identify the problem.\n' + 'Step 2: solve it.\n' + 'Step 3: verify.\n' + ) + full = f'{reasoning_text}\n\nFinal answer.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning == reasoning_text + assert content == '\n\nFinal answer.' + + +class TestExtractReasoningStreaming: + """Streaming ``extract_reasoning_streaming`` tests. + + Each test is parametrized over chunk_size to exercise both fine-grained (token-by-token) and coarse (multi-token) + chunk boundaries. + """ + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_thinking_mode(self, tokenizer, parser, chunk_size): + """Qwen3-8B enable_thinking=True: streaming output matches non- + streaming.""" + reasoning_body = '\nBrief chain of thought.\n' + answer = 'The answer is 42.' + full = f'{reasoning_body}\n\n{answer}' + + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + request = _make_request(stream=True) + r_stream, c_stream = run_reasoning_stream(parser, request, chunks) + + r_ns, c_ns = parser.extract_reasoning(full, _make_request()) + assert r_stream == r_ns + assert c_stream == c_ns + assert answer in c_stream + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_forceful_thinking(self, tokenizer, parser, chunk_size): + """Qwen3-4B-Thinking-2507: no , streaming matches non-streaming.""" + reasoning_body = '\nBrief chain of thought.\n' + answer = 'The answer is 42.' + full = f'{reasoning_body}\n\n{answer}' + + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + request = _make_request(stream=True) + r_stream, c_stream = run_reasoning_stream(parser, request, chunks) + + r_ns, c_ns = parser.extract_reasoning(full, _make_request()) + assert r_stream == r_ns + assert c_stream == c_ns + assert answer in c_stream + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_non_thinking_mode(self, tokenizer, parser, chunk_size): + """Qwen3-8B enable_thinking=False: no tags at all. + + The streaming parser has no way to know that will never arrive, so it treats all text as + reasoning_content. The non-streaming path correctly returns it as content because it can inspect the full + output. This test documents the streaming behavior. + """ + full = 'The answer is 42.' + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + request = _make_request(stream=True) + r_stream, c_stream = run_reasoning_stream(parser, request, chunks) + + assert r_stream == full + assert c_stream == '' + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_empty_reasoning(self, tokenizer, parser, chunk_size): + """Edge case: with empty reasoning body.""" + answer = 'The answer is 42.' + full = f'\n\n{answer}' + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + request = _make_request(stream=True) + r_stream, c_stream = run_reasoning_stream(parser, request, chunks) + + assert r_stream == '' + assert answer in c_stream + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_multiline_reasoning(self, tokenizer, parser, chunk_size): + """Longer reasoning body, streaming matches non-streaming.""" + reasoning_text = ( + '\nStep 1: identify the problem.\n' + 'Step 2: solve it.\n' + 'Step 3: verify.\n' + ) + answer = 'Final answer.' + full = f'{reasoning_text}\n\n{answer}' + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + request = _make_request(stream=True) + r_stream, c_stream = run_reasoning_stream(parser, request, chunks) + + r_ns, c_ns = parser.extract_reasoning(full, _make_request()) + assert r_stream == r_ns + assert c_stream == c_ns + assert answer in c_stream + + +class TestRegistry: + + @pytest.mark.parametrize('name', ['qwen-qwq', 'qwen3', 'intern-s1', 'deepseeek-r1']) + def test_registered_names(self, tokenizer, name): + """All registered aliases resolve to QwenReasoningParser.""" + cls = ReasoningParserManager.get(name) + parser = cls(tokenizer) + assert isinstance(parser, QwenReasoningParser) + + def test_basic_stream_round_trip(self, tokenizer): + """Sanity check: registry-created parser works end-to-end.""" + cls = ReasoningParserManager.get('qwen3') + parser = cls(tokenizer) + full = f'{QwenReasoningParser.start_token}x{QwenReasoningParser.end_token}y' + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=2) + request = _make_request(stream=True) + r_stream, c_stream = run_reasoning_stream(parser, request, chunks) + r_ns, c_ns = parser.extract_reasoning(full, _make_request()) + assert r_stream == r_ns + assert c_stream == c_ns diff --git a/tests/test_lmdeploy/test_qwen3_parser.py b/tests/test_lmdeploy/test_qwen3_parser.py index 5d4529dd21..ec65855e00 100644 --- a/tests/test_lmdeploy/test_qwen3_parser.py +++ b/tests/test_lmdeploy/test_qwen3_parser.py @@ -5,7 +5,7 @@ import pytest import shortuuid -from lmdeploy.serve.openai.reasoning_parser.qwen_qwq_reasoning_parser import QwenQwQReasoningParser +from lmdeploy.serve.openai.reasoning_parser.qwen_qwq_reasoning_parser import QwenReasoningParser from lmdeploy.serve.openai.tool_parser.qwen3_parser import Qwen3ToolParser from lmdeploy.serve.openai.api_server import VariableInterface @@ -211,7 +211,7 @@ def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, Non delta_message.tool_calls = tool_delta.tool_calls delta_message.content = tool_delta.content or '' if VariableInterface.reasoning_parser is not None: - reasoning_delta = VariableInterface.reasoning_parser.extract_reasoning_content_streaming( + reasoning_delta = VariableInterface.reasoning_parser.extract_reasoning_streaming( previous_text=previous_text, current_text=current_text, delta_text=delta_message.content, @@ -252,7 +252,7 @@ def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, Non finish_reason = 'tool_calls' if VariableInterface.reasoning_parser is not None: - reasoning_content, text = VariableInterface.reasoning_parser.extract_reasoning_content(text, request) + reasoning_content, text = VariableInterface.reasoning_parser.extract_reasoning(text, request) choices = [] choice_data = ChatCompletionResponseChoice( @@ -308,7 +308,7 @@ def _stream_parse(request: ChatCompletionRequest, text_sequence: list[str]) -> t def test_parser_stream(text_sequence: list[str], expects: list[TestExpects]): tokenizer = DummyTokenizer() VariableInterface.tool_parser = Qwen3ToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = QwenQwQReasoningParser(tokenizer=tokenizer) + VariableInterface.reasoning_parser = QwenReasoningParser(tokenizer=tokenizer) request = ChatCompletionRequest(model='qwen', messages=[], stream=True) content, reasoning_content, tool_calls = _stream_parse(request, text_sequence) assert len(tool_calls) == len(expects) @@ -328,7 +328,7 @@ def test_parser_stream(text_sequence: list[str], expects: list[TestExpects]): def test_parser_nonstream(text_sequence: list[str], expects: list[TestExpects]): tokenizer = DummyTokenizer() VariableInterface.tool_parser = Qwen3ToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = QwenQwQReasoningParser(tokenizer=tokenizer) + VariableInterface.reasoning_parser = QwenReasoningParser(tokenizer=tokenizer) resp: ChatCompletionResponse = _chat_completion_v1(ChatCompletionRequest(model='qwen', messages=[], stream=False), text_sequence) @@ -358,7 +358,7 @@ def test_no_think_nonstream(): ] tokenizer = DummyTokenizer() VariableInterface.tool_parser = Qwen3ToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = QwenQwQReasoningParser(tokenizer=tokenizer) + VariableInterface.reasoning_parser = QwenReasoningParser(tokenizer=tokenizer) resp: ChatCompletionResponse = _chat_completion_v1(ChatCompletionRequest(model='qwen', messages=[], stream=False), text_sequence) diff --git a/tests/test_lmdeploy/test_qwen3coder_parser.py b/tests/test_lmdeploy/test_qwen3coder_parser.py index 13b4c32603..5ca2079ac7 100644 --- a/tests/test_lmdeploy/test_qwen3coder_parser.py +++ b/tests/test_lmdeploy/test_qwen3coder_parser.py @@ -5,7 +5,6 @@ import pytest import shortuuid -from lmdeploy.serve.openai.tool_parser.qwen3coder_parser import Qwen3CoderToolParser from lmdeploy.serve.openai.api_server import VariableInterface from lmdeploy.serve.openai.protocol import ( @@ -19,6 +18,7 @@ DeltaToolCall, UsageInfo, ) +from lmdeploy.serve.openai.tool_parser.qwen3coder_tool_parser import Qwen3CoderToolParser TestExpects = collections.namedtuple('TestExpects', 'func_name kwargs') @@ -94,7 +94,7 @@ def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, Non delta_message.content = tool_delta.content or '' if VariableInterface.reasoning_parser is not None: parser = VariableInterface.reasoning_parser - reasoning_delta = parser.extract_reasoning_content_streaming(previous_text=previous_text, + reasoning_delta = parser.extract_reasoning_streaming(previous_text=previous_text, current_text=current_text, delta_text=delta_message.content, previous_token_ids=[], @@ -135,7 +135,7 @@ def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, Non if VariableInterface.reasoning_parser is not None: parser = VariableInterface.reasoning_parser - reasoning_content, text = parser.extract_reasoning_content(text, request) + reasoning_content, text = parser.extract_reasoning(text, request) choices = [] choice_data = ChatCompletionResponseChoice( From c516394db6b04a25ad7891eb7251513dbe5192ca Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Thu, 26 Mar 2026 14:32:44 +0000 Subject: [PATCH 05/14] update deepseek reasoning parser ut --- .../test_deepseek_reasoning_parser.py | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py new file mode 100644 index 0000000000..5061d29de3 --- /dev/null +++ b/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py @@ -0,0 +1,128 @@ +# Copyright (c) OpenMMLab. All rights reserved. + +from __future__ import annotations + +import pytest +import transformers +from packaging.version import Version + +from lmdeploy.serve.openai.protocol import ChatCompletionRequest +from lmdeploy.serve.openai.reasoning_parser.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer + +TRANSFORMERS_LT_5 = Version(transformers.__version__) < Version('5.0.0') +REQUIRES_TRANSFORMERS_LT_5 = pytest.mark.skipif( + not TRANSFORMERS_LT_5, + reason=f'requires transformers < 5.0, got {transformers.__version__}', +) +pytestmark = REQUIRES_TRANSFORMERS_LT_5 + + +MODEL_ID = 'deepseek-ai/DeepSeek-V3.1' + +@pytest.fixture(scope='module') +def tokenizer(): + try: + return HuggingFaceTokenizer(MODEL_ID) + except Exception as exc: # noqa: BLE001 + pytest.skip(f'Could not load tokenizer for {MODEL_ID}: {exc}') + + +def _make_request(stream: bool = False) -> ChatCompletionRequest: + return ChatCompletionRequest(model=MODEL_ID, messages=[], stream=stream) + + +def _build_parser(tokenizer: HuggingFaceTokenizer, *, enable_thinking: bool | None) -> DeepSeekV3ReasoningParser: + return DeepSeekV3ReasoningParser(tokenizer, enable_thinking=enable_thinking) + + +def simulate_pipeline_chunks( + tokenizer: HuggingFaceTokenizer, + full_text: str, + *, + chunk_size: int = 1, + skip_special_tokens: bool = True, + spaces_between_special_tokens: bool = True, +) -> list[tuple[str, list[int]]]: + all_ids = tokenizer.encode(full_text, add_bos=False, add_special_tokens=False) + state = DetokenizeState(0) + accumulated: list[int] = [] + chunks: list[tuple[str, list[int]]] = [] + offset = 0 + while offset < len(all_ids): + accumulated.extend(all_ids[offset:offset + chunk_size]) + offset += chunk_size + ids_offset_before = state.ids_offset + delta_text, state = tokenizer.detokenize_incrementally( + accumulated, + state, + skip_special_tokens=skip_special_tokens, + spaces_between_special_tokens=spaces_between_special_tokens, + ) + delta_ids = accumulated[ids_offset_before:len(accumulated)] + chunks.append((delta_text, delta_ids)) + return chunks + + +def run_reasoning_stream( + parser: DeepSeekV3ReasoningParser, + request: object, + chunks: list[tuple[str, list[int]]], +) -> tuple[str, str]: + state = get_streaming_state(request) + reasoning_acc = '' + content_acc = '' + for delta_text, delta_ids in chunks: + state.update(delta_text, delta_ids) + delta_msg = parser.extract_reasoning_streaming( + delta_text=delta_text or '', + delta_token_ids=delta_ids, + request=request, + ) + if delta_msg is not None: + if delta_msg.reasoning_content: + reasoning_acc += delta_msg.reasoning_content + if delta_msg.content is not None: + content_acc += delta_msg.content + state.step() + return reasoning_acc, content_acc + + +class TestExtractReasoning: + + def test_enable_thinking_true(self, tokenizer): + parser = _build_parser(tokenizer, enable_thinking=True) + full = '\nBrief chain of thought.\n\n\nThe answer is 42.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning == '\nBrief chain of thought.\n' + assert content == '\n\nThe answer is 42.' + + def test_enable_thinking_none(self, tokenizer): + parser = _build_parser(tokenizer, enable_thinking=None) + full = 'The answer is 42.' + reasoning, content = parser.extract_reasoning(full, _make_request()) + assert reasoning is None + assert content == full + + +class TestExtractReasoningStreaming: + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_enable_thinking_true(self, tokenizer, chunk_size): + parser = _build_parser(tokenizer, enable_thinking=True) + full = '\nBrief chain of thought.\n\n\nThe answer is 42.' + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + r_stream, c_stream = run_reasoning_stream(parser, _make_request(stream=True), chunks) + r_ns, c_ns = parser.extract_reasoning(full, _make_request()) + assert r_stream == r_ns + assert c_stream == c_ns + + @pytest.mark.parametrize('chunk_size', [1, 3]) + def test_enable_thinking_none(self, tokenizer, chunk_size): + parser = _build_parser(tokenizer, enable_thinking=False) + full = 'The answer is 42.' + chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) + r_stream, c_stream = run_reasoning_stream(parser, _make_request(stream=True), chunks) + assert r_stream == '' + assert c_stream == full From d3eb9738f0af93d60665f15f50ca1a02340ac1b9 Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Mon, 30 Mar 2026 13:31:56 +0000 Subject: [PATCH 06/14] agent's first refactor version --- lmdeploy/serve/openai/api_server.py | 101 ++-- lmdeploy/serve/openai/harmony_utils.py | 104 +---- .../serve/openai/reasoning_parser/__init__.py | 15 +- .../deepseek_v3_reasoning_parser.py | 4 + .../gpt_oss_reasoning_parser.py | 145 ++++++ .../identity_reasoning_parser.py | 3 + .../reasoning_parser/qwen_reasoning_parser.py | 20 +- .../reasoning_parser/reasoning_parser.py | 63 +-- lmdeploy/serve/openai/response_parser.py | 167 +++++++ .../tool_parser/internlm2_tool_parser.py | 20 +- .../openai/tool_parser/llama3_tool_parser.py | 18 +- .../openai/tool_parser/qwen2d5_tool_parser.py | 20 +- .../openai/tool_parser/qwen3_tool_parser.py | 180 +++---- .../tool_parser/qwen3coder_tool_parser.py | 95 ++-- .../serve/openai/tool_parser/tool_parser.py | 22 +- .../test_deepseek_reasoning_parser.py | 5 +- .../test_harmony_gpt_oss_parser.py | 0 .../test_qwen_reasoning_parser.py | 6 +- .../server/tool_parsers/test_qwen3_parser.py | 441 ++++++++++++++++++ .../tool_parsers}/test_qwen3coder_parser.py | 30 +- tests/test_lmdeploy/test_qwen3_parser.py | 368 --------------- 21 files changed, 1061 insertions(+), 766 deletions(-) create mode 100644 lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py create mode 100644 lmdeploy/serve/openai/response_parser.py rename tests/test_lmdeploy/{ => server/reasoning_parsers}/test_harmony_gpt_oss_parser.py (100%) create mode 100644 tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py rename tests/test_lmdeploy/{ => server/tool_parsers}/test_qwen3coder_parser.py (94%) delete mode 100644 tests/test_lmdeploy/test_qwen3_parser.py diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 664fccea6e..cca5111e06 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -40,7 +40,6 @@ MigrationRequest, ) from lmdeploy.serve.core import AsyncEngine -from lmdeploy.serve.openai.harmony_utils import GptOssChatParser from lmdeploy.serve.openai.protocol import ( AbortRequest, ChatCompletionRequest, @@ -74,12 +73,10 @@ UpdateParamsRequest, UsageInfo, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ( - ReasoningParser, - ReasoningParserManager, - get_streaming_state, -) -from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParser, ToolParserManager +from lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser import GptOssReasoningParser +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager +from lmdeploy.serve.openai.response_parser import ResponseParser +from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParserManager from lmdeploy.serve.utils.server_utils import validate_json_request from lmdeploy.tokenizer import DetokenizeState, Tokenizer from lmdeploy.utils import get_logger @@ -96,10 +93,6 @@ class VariableInterface: # following are for registering to proxy server proxy_url: str | None = None api_server_url: str | None = None - # following are for reasoning parsers - reasoning_parser_cls: type[ReasoningParser] | None = None - # following is for tool parsers - tool_parser_cls: type[ToolParser] | None = None allow_terminate_by_client: bool = False enable_abort_handling: bool = False @@ -413,8 +406,6 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque error_check_ret = check_request(request) if error_check_ret is not None: return error_check_ret - if VariableInterface.tool_parser is not None: - request = VariableInterface.tool_parser.adjust_request(request) session = VariableInterface.get_session(request.session_id) json_request = await raw_request.json() @@ -430,13 +421,20 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque adapter_name = model_name # got a adapter name request_id = str(session.session_id) created_time = int(time.time()) - gpt_oss_parser = None - if VariableInterface.async_engine.arch == 'GptOssForCausalLM': - gpt_oss_parser = GptOssChatParser() if isinstance(request.stop, str): request.stop = [request.stop] + tokenizer = VariableInterface.async_engine.tokenizer.model + response_parser = ResponseParser(request=request, tokenizer=tokenizer) + + # Harmony GPT-OSS: explicit `--reasoning-parser gpt-oss`, or GptOssForCausalLM arch. + gpt_oss_parser = None + if isinstance(response_parser.reasoning_parser, GptOssReasoningParser): + gpt_oss_parser = response_parser.reasoning_parser + elif VariableInterface.async_engine.arch == 'GptOssForCausalLM': + gpt_oss_parser = GptOssReasoningParser(tokenizer, **response_parser._kwargs) + gen_logprobs, logits_processors = None, None if request.logprobs and request.top_logprobs: gen_logprobs = request.top_logprobs @@ -447,7 +445,7 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque if request.logit_bias is not None: try: logits_processors = [ - logit_bias_logits_processor(request.logit_bias, VariableInterface.async_engine.tokenizer.model) + logit_bias_logits_processor(request.logit_bias, tokenizer) ] except Exception as e: return create_error_response(HTTPStatus.BAD_REQUEST, str(e)) @@ -508,7 +506,7 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque chat_template_kwargs['enable_thinking'] = request.enable_thinking else: logger.warning('`enable_thinking` in `chat_template_kwargs` will override the value in request.') - enable_thinking = chat_template_kwargs.get('enable_thinking', None) + result_generator = VariableInterface.async_engine.generate( request.messages, session, @@ -544,17 +542,8 @@ def create_stream_response_json(index: int, return response_json - tokenizer = VariableInterface.async_engine.tokenizer - reasoning_parser, tool_parser = None, None - if VariableInterface.reasoning_parser_cls is not None: - reasoning_parser = VariableInterface.reasoning_parser_cls(tokenizer, **chat_template_kwargs) - if VariableInterface.tool_parser_cls is not None: - tool_parser = VariableInterface.tool_parser_cls(tokenizer, **chat_template_kwargs) - async def completion_stream_generator() -> AsyncGenerator[str, None]: streaming_tools = False - # Shared state for streaming parsers (previous/current text & token ids) - parser_state = get_streaming_state(request) if reasoning_parser or tool_parser else None async for res in result_generator: logprobs, usage = None, None if gen_logprobs and res.logprobs: @@ -574,30 +563,23 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: if res.finish_reason == 'stop' and len(delta_message.tool_calls) > 0: res.finish_reason = 'tool_calls' else: - delta_message = DeltaMessage(role='assistant', content=res.response) - if parser_state is not None: - parser_state.update(res.response, delta_token_ids) - if request.tool_choice != 'none' and tool_parser is not None: + if response_parser is not None: + delta_message, tool_emitted = response_parser.stream_chunk( + res.response, + delta_token_ids + ) + if tool_emitted: + streaming_tools = True + else: + delta_message = DeltaMessage(role='assistant', content=res.response) + + if (request.tool_choice != 'none' and response_parser is not None + and response_parser.tool_parser is not None): if res.finish_reason == 'stop' and streaming_tools is True: res.finish_reason = 'tool_calls' - tool_delta = tool_parser.extract_tool_calls_streaming( - delta_text=delta_message.content, delta_token_ids=delta_token_ids, request=request) - if tool_delta is not None: - delta_message.tool_calls = tool_delta.tool_calls - delta_message.content = tool_delta.content - if isinstance(tool_delta.tool_calls, list) and len(tool_delta.tool_calls): - streaming_tools = True - elif (request.tool_choice != 'none' and request.tools is not None - and tool_parser is None): - logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') - if reasoning_parser and enable_thinking is not False: - reasoning_delta = reasoning_parser.extract_reasoning_streaming( - delta_text=delta_message.content or '', delta_token_ids=delta_token_ids, request=request) - if reasoning_delta is not None: - delta_message.reasoning_content = reasoning_delta.reasoning_content - delta_message.content = reasoning_delta.content - if parser_state is not None: - parser_state.step() + elif request.tool_choice != 'none' and request.tools is not None: + if ResponseParser.tool_parser_cls is None: + logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') if request.return_token_ids: delta_message.gen_tokens = delta_token_ids response_json = create_stream_response_json(index=0, @@ -643,10 +625,10 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: else: tool_calls = None reasoning_content = None - if request.tool_choice != 'none' and tool_parser is not None: + if response_parser is not None: try: - tool_call_info = tool_parser.extract_tool_calls(text, request=request) - text, tool_calls = tool_call_info.content, tool_call_info.tool_calls + text, tool_calls, reasoning_content = response_parser.parse_complete( + text) if isinstance(tool_calls, list) and len(tool_calls): if final_res.finish_reason == 'stop': final_res.finish_reason = 'tool_calls' @@ -654,11 +636,9 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: except Exception as e: logger.error(f'Failed to parse {text}. Exception: {e}.') return create_error_response(HTTPStatus.BAD_REQUEST, 'Failed to parse fc related info to json format!') - elif request.tool_choice != 'none' and request.tools is not None and tool_parser is None: - logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') - - if reasoning_parser and enable_thinking is not False: - reasoning_content, text = reasoning_parser.extract_reasoning(text, request) + elif request.tool_choice != 'none' and request.tools is not None: + if ResponseParser.tool_parser_cls is None: + logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') message = ChatMessage(role='assistant', content=text, @@ -1322,17 +1302,18 @@ async def dispatch(self, request: Request, call_next): def set_parsers(reasoning_parser_name: str | None = None, tool_parser_name: str | None = None, **kwargs): - """Set tool parser and reasoning parsers.""" + """Set tool parser and reasoning parser types on + :class:`~lmdeploy.serve.openai.response_parser.ResponseParser`.""" if reasoning_parser_name is not None: if reasoning_parser_name in ReasoningParserManager.module_dict: - VariableInterface.reasoning_parser_cls = ReasoningParserManager.get(reasoning_parser_name) + ResponseParser.reasoning_parser_cls = ReasoningParserManager.get(reasoning_parser_name) else: raise ValueError(f'The reasoning parser {reasoning_parser_name} is not in the parser list: ' f'{ReasoningParserManager.module_dict.keys()}') if tool_parser_name is not None: if tool_parser_name in ToolParserManager.module_dict: - VariableInterface.tool_parser_cls = ToolParserManager.get(tool_parser_name) + ResponseParser.tool_parser_cls = ToolParserManager.get(tool_parser_name) else: raise ValueError(f'The tool parser {tool_parser_name} is not in the parser list: ' f'{ToolParserManager.module_dict.keys()}') diff --git a/lmdeploy/serve/openai/harmony_utils.py b/lmdeploy/serve/openai/harmony_utils.py index 2810725c0f..1b35aa8eff 100644 --- a/lmdeploy/serve/openai/harmony_utils.py +++ b/lmdeploy/serve/openai/harmony_utils.py @@ -1,94 +1,16 @@ # Copyright (c) OpenMMLab. All rights reserved. -# Modified from https://github.com/vllm-project/vllm/blob/v0.10.2rc1/vllm/entrypoints/harmony_utils.py - -import shortuuid -from openai_harmony import HarmonyEncodingName, Role, StreamableParser, load_harmony_encoding - -from lmdeploy.serve.openai.protocol import ( - ChatMessage, - DeltaFunctionCall, - DeltaMessage, - DeltaToolCall, - FunctionCall, - ToolCall, +"""Backward-compatible re-exports for Harmony GPT-OSS helpers. + +Prefer importing from :mod:`lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser`. +""" +from lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser import ( + GptOssChatParser, + get_encoding, + get_streamable_parser_for_assistant, ) -_harmony_encoding = None - - -def get_encoding(): - global _harmony_encoding - if _harmony_encoding is None: - _harmony_encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS) - return _harmony_encoding - - -def get_streamable_parser_for_assistant() -> 'StreamableParser': - return StreamableParser(get_encoding(), role=Role.ASSISTANT) - - -class GptOssChatParser: - - def __init__(self): - self.parser = get_streamable_parser_for_assistant() - - def parse_streaming(self, tokens: list[int]) -> DeltaMessage: - parser = self.parser - delta_message = DeltaMessage(role='assistant') - content = '' - reasoning_content = '' - tool_calls = [] - delta_tool_call = None - for token in tokens: - prev_recipient = parser.current_recipient - parser.process(token) - cur_channel = parser.current_channel - cur_recipient = parser.current_recipient - delta_text = parser.last_content_delta or '' - if cur_channel == 'final': - content += delta_text - elif cur_channel == 'analysis': - reasoning_content += delta_text - elif cur_channel == 'commentary' and cur_recipient and cur_recipient.startswith('functions.'): - base_index = 0 - for msg in parser.messages: - if msg.channel == 'commentary' and msg.recipient and msg.recipient.startswith('functions.'): - base_index += 1 - if prev_recipient != cur_recipient: - if delta_tool_call is not None: - tool_calls.append(delta_tool_call) - tool_name = cur_recipient.split('functions.', 1)[1] - delta_tool_call = DeltaToolCall(id=f'chatcmpl-tool-{shortuuid.random()}', - type='function', - index=base_index, - function=DeltaFunctionCall(name=tool_name, arguments='')) - elif delta_text: - # Continuing the same tool call. Ensure we don't duplicate the - # very first delta string in this chunk. Previously we initialized - # with arguments=delta_text and then appended again, causing - # duplicated content like "locationlocation". - if delta_tool_call is None: - # We are in the middle of a tool call carried over from the - # previous chunk. Initialize an empty arguments buffer. - delta_tool_call = DeltaToolCall(index=base_index, function=DeltaFunctionCall(arguments='')) - delta_tool_call.function.arguments += delta_text - - if delta_tool_call: - tool_calls.append(delta_tool_call) - - delta_message.content = content if content else None - delta_message.reasoning_content = reasoning_content if reasoning_content else None - delta_message.tool_calls = tool_calls - return delta_message - - def parse_full(self, tokens: list[int]) -> ChatMessage: - delta_message = self.parse_streaming(tokens) - tool_calls = [] - for delta_tool_call in delta_message.tool_calls: - function = FunctionCall(**delta_tool_call.function.model_dump()) - tool_calls.append(ToolCall(id=delta_tool_call.id, type=delta_tool_call.type, function=function)) - chat_message = ChatMessage(role='assistant', - content=delta_message.content, - tool_calls=tool_calls, - reasoning_content=delta_message.reasoning_content) - return chat_message +__all__ = [ + 'GptOssChatParser', + 'get_encoding', + 'get_streamable_parser_for_assistant', +] diff --git a/lmdeploy/serve/openai/reasoning_parser/__init__.py b/lmdeploy/serve/openai/reasoning_parser/__init__.py index b26208ba2a..6e6f1072be 100644 --- a/lmdeploy/serve/openai/reasoning_parser/__init__.py +++ b/lmdeploy/serve/openai/reasoning_parser/__init__.py @@ -1,24 +1,27 @@ # Copyright (c) OpenMMLab. All rights reserved. +from lmdeploy.serve.openai.response_parser import StreamBuffer + from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser from .deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser +from .gpt_oss_reasoning_parser import GptOssReasoningParser from .identity_reasoning_parser import IdentityReasoningParser from .qwen_reasoning_parser import QwenReasoningParser from .reasoning_parser import ( - ReasoningParser, - ReasoningParserManager, - StreamingParserState, - ThinkingReasoningParser, - get_streaming_state, + ReasoningParser, + ReasoningParserManager, + StreamingParserState, + ThinkingReasoningParser, ) __all__ = [ 'ReasoningParser', 'ReasoningParserManager', + 'StreamBuffer', 'StreamingParserState', 'ThinkingReasoningParser', - 'get_streaming_state', 'DeepSeekR1ReasoningParser', 'QwenReasoningParser', 'IdentityReasoningParser', 'DeepSeekV3ReasoningParser', + 'GptOssReasoningParser', ] diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py index eecb96d8d6..f9eaec03a8 100644 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING from lmdeploy.serve.openai.protocol import DeltaMessage +from lmdeploy.serve.openai.response_parser import StreamBuffer from .identity_reasoning_parser import IdentityReasoningParser from .reasoning_parser import ReasoningParser @@ -39,11 +40,14 @@ def extract_reasoning_streaming( delta_text: str, delta_token_ids: list[int], request: object, + *, + stream_buffer: StreamBuffer, **kwargs, ) -> DeltaMessage | None: return self._parser.extract_reasoning_streaming( delta_text, delta_token_ids, request, + stream_buffer=stream_buffer, **kwargs, ) diff --git a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py new file mode 100644 index 0000000000..9301f868aa --- /dev/null +++ b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py @@ -0,0 +1,145 @@ +# Copyright (c) OpenMMLab. All rights reserved. +# Modified from https://github.com/vllm-project/vllm/blob/v0.10.2rc1/vllm/entrypoints/harmony_utils.py +from __future__ import annotations + +import shortuuid +from openai_harmony import HarmonyEncodingName, Role, StreamableParser, load_harmony_encoding + +from lmdeploy.serve.openai.protocol import ( + ChatCompletionRequest, + ChatMessage, + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + FunctionCall, + ToolCall, +) +from lmdeploy.serve.openai.response_parser import StreamBuffer + +from .reasoning_parser import ReasoningParser, ReasoningParserManager + +_harmony_encoding = None + + +def get_encoding(): + global _harmony_encoding + if _harmony_encoding is None: + _harmony_encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS) + return _harmony_encoding + + +def get_streamable_parser_for_assistant() -> StreamableParser: + return StreamableParser(get_encoding(), role=Role.ASSISTANT) + + +class GptOssChatParser: + """Harmony stream parser for GPT-OSS (assistant role): content, reasoning, + tool calls.""" + + def __init__(self): + self.parser = get_streamable_parser_for_assistant() + + def parse_streaming(self, tokens: list[int]) -> DeltaMessage: + parser = self.parser + delta_message = DeltaMessage(role='assistant') + content = '' + reasoning_content = '' + tool_calls = [] + delta_tool_call = None + for token in tokens: + prev_recipient = parser.current_recipient + parser.process(token) + cur_channel = parser.current_channel + cur_recipient = parser.current_recipient + delta_text = parser.last_content_delta or '' + if cur_channel == 'final': + content += delta_text + elif cur_channel == 'analysis': + reasoning_content += delta_text + elif cur_channel == 'commentary' and cur_recipient and cur_recipient.startswith('functions.'): + base_index = 0 + for msg in parser.messages: + if msg.channel == 'commentary' and msg.recipient and msg.recipient.startswith('functions.'): + base_index += 1 + if prev_recipient != cur_recipient: + if delta_tool_call is not None: + tool_calls.append(delta_tool_call) + tool_name = cur_recipient.split('functions.', 1)[1] + delta_tool_call = DeltaToolCall(id=f'chatcmpl-tool-{shortuuid.random()}', + type='function', + index=base_index, + function=DeltaFunctionCall(name=tool_name, arguments='')) + elif delta_text: + # Continuing the same tool call. Ensure we don't duplicate the + # very first delta string in this chunk. Previously we initialized + # with arguments=delta_text and then appended again, causing + # duplicated content like "locationlocation". + if delta_tool_call is None: + # We are in the middle of a tool call carried over from the + # previous chunk. Initialize an empty arguments buffer. + delta_tool_call = DeltaToolCall(index=base_index, function=DeltaFunctionCall(arguments='')) + delta_tool_call.function.arguments += delta_text + + if delta_tool_call: + tool_calls.append(delta_tool_call) + + delta_message.content = content if content else None + delta_message.reasoning_content = reasoning_content if reasoning_content else None + delta_message.tool_calls = tool_calls + return delta_message + + def parse_full(self, tokens: list[int]) -> ChatMessage: + delta_message = self.parse_streaming(tokens) + tool_calls = [] + for delta_tool_call in delta_message.tool_calls: + function = FunctionCall(**delta_tool_call.function.model_dump()) + tool_calls.append(ToolCall(id=delta_tool_call.id, type=delta_tool_call.type, function=function)) + chat_message = ChatMessage(role='assistant', + content=delta_message.content, + tool_calls=tool_calls, + reasoning_content=delta_message.reasoning_content) + return chat_message + + +@ReasoningParserManager.register_module('gpt-oss') +class GptOssReasoningParser(ReasoningParser): + """Reasoning / channel parser for OpenAI Harmony GPT-OSS wire format (token + stream). + + Use ``--reasoning-parser gpt-oss`` when serving GPT-OSS models. When the engine + architecture is ``GptOssForCausalLM``, the API server also enables this parser + automatically even if the flag is omitted. + """ + + def __init__(self, tokenizer: object, **kwargs): + super().__init__(tokenizer, **kwargs) + self._chat = GptOssChatParser() + + def parse_streaming(self, tokens: list[int]) -> DeltaMessage: + """Parse one engine chunk of token ids into a + :class:`~lmdeploy.serve.openai.protocol.DeltaMessage`.""" + return self._chat.parse_streaming(tokens) + + def parse_full(self, tokens: list[int]) -> ChatMessage: + """Parse the full completion token sequence into a + :class:`~lmdeploy.serve.openai.protocol.ChatMessage`.""" + return self._chat.parse_full(tokens) + + def extract_reasoning_streaming( + self, + delta_text: str, + delta_token_ids: list[int], + request: object, + *, + stream_buffer: StreamBuffer, + **kwargs, + ): + """Not used; GPT-OSS uses :meth:`parse_streaming` on token ids in the + API server.""" + return None + + def extract_reasoning(self, model_output: str, request: + ChatCompletionRequest, **kwargs) -> tuple[str | None, str | None]: + """Not used for Harmony decoding; non-streaming path uses + :meth:`parse_full` on token ids.""" + return None, model_output diff --git a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py index f0c818327c..cc14868308 100644 --- a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py @@ -5,6 +5,7 @@ from lmdeploy.serve.openai.protocol import DeltaMessage from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser +from lmdeploy.serve.openai.response_parser import StreamBuffer if TYPE_CHECKING: from lmdeploy.serve.openai.protocol import ChatCompletionRequest @@ -26,6 +27,8 @@ def extract_reasoning_streaming( delta_text: str, delta_token_ids: list[int], request: object, + *, + stream_buffer: StreamBuffer, **kwargs, ) -> DeltaMessage | None: # Just wrap delta_text as content, ignore reasoning diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py index bf041de428..261360d537 100644 --- a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py @@ -1,10 +1,14 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/qwen3_reasoning_parser.py +from typing import TYPE_CHECKING from lmdeploy.serve.openai.protocol import DeltaMessage +from lmdeploy.serve.openai.response_parser import StreamBuffer -from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser, get_streaming_state +from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser +if TYPE_CHECKING: + pass @ReasoningParserManager.register_module(name=['qwen-qwq', 'qwen3', 'intern-s1', 'deepseeek-r1']) class QwenReasoningParser(ThinkingReasoningParser): @@ -21,10 +25,16 @@ class QwenReasoningParser(ThinkingReasoningParser): start_token = '' end_token = '' - def extract_reasoning_streaming(self, delta_text: str, delta_token_ids: list[int], - request: object, **kwargs) -> DeltaMessage | None: - state = get_streaming_state(request) - previous_token_ids = state.previous_token_ids + def extract_reasoning_streaming( + self, + delta_text: str, + delta_token_ids: list[int], + request: object, + *, + stream_buffer: StreamBuffer, + **kwargs, + ) -> DeltaMessage | None: + previous_token_ids = stream_buffer.previous_token_ids # Strip from delta if present (old template / edge case where the model generates itself). if self.start_token_id in delta_token_ids: diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index 7de8cf71a6..95c03dea9d 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -1,45 +1,15 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/reasoning_parsers -from dataclasses import dataclass, field from functools import cached_property from mmengine import Registry from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage +from lmdeploy.serve.openai.response_parser import StreamBuffer ReasoningParserManager = Registry('reasoning_parser', locations=['lmdeploy.serve.openai.reasoning_parser']) - -@dataclass -class StreamingParserState: - """Shared state for streaming parsing, attached to a request object. - - Both reasoning parsers and tool parsers read/write the same state so that text accumulated by the streaming loop is - available to all parsers without duplication. - """ - previous_text: str = '' - current_text: str = '' - previous_token_ids: list[int] = field(default_factory=list) - current_token_ids: list[int] = field(default_factory=list) - - def update(self, delta_text: str, delta_token_ids: list[int]) -> None: - """Accumulate new delta into current_text / current_token_ids.""" - self.current_text += delta_text - self.current_token_ids.extend(delta_token_ids) - - def step(self) -> None: - """Advance: copy current -> previous (call at end of each iteration).""" - self.previous_text = self.current_text - self.previous_token_ids = self.current_token_ids - - -def get_streaming_state(request: object) -> StreamingParserState: - """Get or create a StreamingParserState on the request object.""" - state = getattr(request, '_streaming_parser_state', None) - if state is None: - state = StreamingParserState() - setattr(request, '_streaming_parser_state', state) - return state +StreamingParserState = StreamBuffer class ReasoningParser: @@ -59,6 +29,8 @@ def extract_reasoning_streaming( delta_text: str, delta_token_ids: list[int], request: object, + *, + stream_buffer: StreamBuffer, **kwargs, ) -> DeltaMessage | None: """Instance method that should be implemented for extracting reasoning @@ -69,9 +41,10 @@ def extract_reasoning_streaming( delta_text: The new text chunk (may have been modified by the tool parser before being passed here). delta_token_ids: The new token ids for this chunk. - request: The request object; a ``StreamingParserState`` is attached - to it via ``get_streaming_state(request)`` so that previous / - current text and token ids are available. + request: The request object. + stream_buffer: Cumulative decoding state (``ResponseParser.stream``); + Token ids from prior chunks are in ``stream_buffer.previous_token_ids`` + at the time this method runs (after ``stream_buffer.update`` for this chunk). Returns a DeltaMessage with reasoning_content and/or content fields, or None if the delta should be skipped. @@ -129,6 +102,8 @@ def extract_reasoning_streaming( delta_text: str, delta_token_ids: list[int], request: object, + *, + stream_buffer: StreamBuffer, **kwargs, ) -> DeltaMessage | None: """Extract reasoning content from a streaming model-generated string. @@ -137,15 +112,13 @@ def extract_reasoning_streaming( delta_text: The new text chunk (may have been modified by the tool parser before being passed here). delta_token_ids: The new token ids for this chunk. - request: The request object; a ``StreamingParserState`` is attached - to it via ``get_streaming_state(request)`` so that previous / - current text and token ids are available. + request: The request object. + stream_buffer: Cumulative decoding state (see base class). Returns a DeltaMessage with reasoning_content and/or content fields, or None if the delta should be skipped. """ - state = get_streaming_state(request) - previous_token_ids = state.previous_token_ids + previous_token_ids = stream_buffer.previous_token_ids # Handle single special tokens if len(delta_token_ids) == 1 and (delta_token_ids[0] in [self.start_token_id, self.end_token_id]): @@ -192,8 +165,10 @@ def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', Returns: A tuple of (reasoning_content, final_output). Either may be None. """ - # Check if the start token is present in the model output, remove it - # if it is present. + + if self.start_token not in model_output and self.end_token not in model_output: + return None, model_output + model_output_parts = model_output.partition(self.start_token) model_output = ( model_output_parts[2] if model_output_parts[1] else model_output_parts[0] @@ -205,6 +180,8 @@ def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', return model_output, None else: reasoning, _, content = model_output.partition(self.end_token) - # If generation stops right after end-of-think, return null content + # If generation stops right after end-of-think, return None content final_content = content or None + # If the model_output is like "...", return None reasoning + reasoning = reasoning or None return reasoning, final_content diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py new file mode 100644 index 0000000000..8d66fa849e --- /dev/null +++ b/lmdeploy/serve/openai/response_parser.py @@ -0,0 +1,167 @@ +# Copyright (c) OpenMMLab. All rights reserved. +"""Unified streaming accumulation and façade for reasoning + tool call +parsing.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, ClassVar + +from transformers import PreTrainedTokenizerBase + +from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage +from lmdeploy.utils import get_logger + +if TYPE_CHECKING: + from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser + from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParser + +logger = get_logger(__name__) + + +@dataclass +class StreamBuffer: + """Cumulative decode snapshot (``ResponseParser.stream_buffer``); also + passed as ``stream_buffer=``.""" + + previous_text: str = '' + current_text: str = '' + previous_token_ids: list[int] = field(default_factory=list) + current_token_ids: list[int] = field(default_factory=list) + + def update(self, delta_text: str, delta_token_ids: list[int]) -> None: + self.current_text += delta_text + self.current_token_ids.extend(delta_token_ids) + + def step(self) -> None: + self.previous_text = self.current_text + self.previous_token_ids = self.current_token_ids + + +class ResponseParser: + """Single entry for streaming / complete post-processing (tool then + reasoning). + + Parser *types* are configured at process start via :func:`lmdeploy.serve.openai.api_server.set_parsers`, + which sets the class attributes below. Tests may assign those attributes on a subclass or temporarily on + ``ResponseParser`` before construction. + + Streaming text/token accumulation lives on this instance (``current_text``, ``previous_token_ids``, etc.). + """ + + reasoning_parser_cls: ClassVar[type[ReasoningParser] | None] = None + tool_parser_cls: ClassVar[type[ToolParser] | None] = None + + @classmethod + def chat_template_kwargs_from_request(cls, request: ChatCompletionRequest) -> dict: + """Merge ``request.enable_thinking`` into ``chat_template_kwargs`` + (deprecated field path).""" + chat_template_kwargs = request.chat_template_kwargs or {} + if request.enable_thinking is not None: + logger.warning('`enable_thinking` will be deprecated in the future, ' + 'please use `chat_template_kwargs` instead.') + if chat_template_kwargs.get('enable_thinking') is None: + chat_template_kwargs['enable_thinking'] = request.enable_thinking + else: + logger.warning( + '`enable_thinking` in `chat_template_kwargs` will override the value in request.') + return chat_template_kwargs + + def __init__( + self, + request: ChatCompletionRequest, + tokenizer: PreTrainedTokenizerBase, + ): + self._kwargs = type(self).chat_template_kwargs_from_request(request) + self.enable_thinking: bool | None = self._kwargs.get('enable_thinking', None) + rcls = type(self).reasoning_parser_cls + tcls = type(self).tool_parser_cls + self.reasoning_parser: ReasoningParser | None = ( + rcls(tokenizer, **self._kwargs) if rcls else None + ) + self.tool_parser: ToolParser | None = ( + tcls(tokenizer, **self._kwargs) if tcls else None + ) + if self.tool_parser is not None: + self.request = self.tool_parser.adjust_request(request) + else: + self.request = request + self.stream_buffer = StreamBuffer() + + def _stream_update(self, delta_text: str, delta_token_ids: list[int]) -> None: + self.stream_buffer.update(delta_text, delta_token_ids) + + def _stream_step(self) -> None: + self.stream_buffer.step() + + def stream_chunk( + self, + delta_text: str, + delta_token_ids: list[int], + **kwargs, + ) -> tuple[DeltaMessage, bool]: + """Update state, run tool then reasoning parsers. + + Returns: + (delta_message, tool_calls_emitted) — the latter is True if this chunk + carries non-empty ``tool_calls`` (for finish_reason handling). + """ + req = self.request + self._stream_update(delta_text, delta_token_ids) + + delta_message = DeltaMessage(role='assistant', content=None) + tool_calls_emitted = False + + if req.tool_choice != 'none' and self.tool_parser is not None: + tool_delta = self.tool_parser.extract_tool_calls_streaming( + delta_text=delta_text, + delta_token_ids=delta_token_ids, + request=req, + stream_buffer=self.stream_buffer, + **kwargs, + ) + if tool_delta is not None: + if tool_delta.tool_calls is not None: + delta_message.tool_calls = tool_delta.tool_calls + if tool_delta.content is not None: + delta_message.content = tool_delta.content + if isinstance(tool_delta.tool_calls, list) and len(tool_delta.tool_calls): + tool_calls_emitted = True + elif req.tool_choice != 'none' and req.tools is not None and self.tool_parser is None: + pass # caller logs error + + if self.reasoning_parser is not None and self.enable_thinking is not False: + reasoning_delta = self.reasoning_parser.extract_reasoning_streaming( + delta_text=delta_message.content or '', + delta_token_ids=delta_token_ids, + request=req, + stream_buffer=self.stream_buffer, + **kwargs, + ) + if reasoning_delta is not None: + delta_message.reasoning_content = reasoning_delta.reasoning_content + delta_message.content = reasoning_delta.content + + self._stream_step() + return delta_message, tool_calls_emitted + + def parse_complete( + self, + text: str, + **kwargs, + ) -> tuple[str, list | None, str | None]: + """Non-streaming: strip tools then reasoning. Returns (text, tool_calls, reasoning_content).""" + req = self.request + tool_calls = None + reasoning_content = None + out_text = text + + if req.tool_choice != 'none' and self.tool_parser is not None: + tool_call_info = self.tool_parser.extract_tool_calls(out_text, request=req) + out_text, tool_calls = tool_call_info.content, tool_call_info.tool_calls + elif req.tool_choice != 'none' and req.tools is not None and self.tool_parser is None: + pass + + if self.reasoning_parser is not None and self.enable_thinking is not False: + reasoning_content, out_text = self.reasoning_parser.extract_reasoning(out_text, req) + + return out_text, tool_calls, reasoning_content diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index aa02feed6b..d79ecfc267 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -15,7 +15,7 @@ FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -29,7 +29,11 @@ class Internlm2ToolParser(ToolParser): def __init__(self, tokenizer: object): super().__init__(tokenizer) - self.position = 0 + self.parse_cursor = 0 + self.current_tool_id = -1 + self.current_tool_name_sent = False + self.streamed_args_for_tool: list[str] = [] + self.prev_tool_call_arr: list[dict] = [] def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: if request.tools and request.tool_choice != 'none': @@ -51,18 +55,20 @@ def extract_tool_calls_streaming( delta_text: str, delta_token_ids: Sequence[int], request: ChatCompletionRequest, + *, + stream_buffer: StreamBuffer, + **kwargs, ) -> DeltaMessage | None: - state = get_streaming_state(request) - current_text = state.current_text + current_text = stream_buffer.current_text if '<|action_start|>' not in current_text: - self.position = len(current_text) + self.parse_cursor = len(current_text) return DeltaMessage(content=delta_text) # if the tool call is sended, return a empty delta message # to make sure the finish_reason will be send correctly. if self.current_tool_id > 0: return DeltaMessage(content='') - last_pos = self.position + last_pos = self.parse_cursor if '<|action_start|><|plugin|>\n' not in current_text[last_pos:]: return None @@ -70,7 +76,7 @@ def extract_tool_calls_streaming( text, action = new_delta.split('<|action_start|><|plugin|>\n') if len(text) > 0: - self.position = self.position + len(text) + self.parse_cursor = self.parse_cursor + len(text) return DeltaMessage(content=text) action = action.strip() diff --git a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py index d3f224b958..7d288736fe 100644 --- a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py @@ -16,7 +16,7 @@ FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -35,13 +35,11 @@ class Llama3JsonToolParser(ToolParser): def __init__(self, tokenizer: object): super().__init__(tokenizer) - - # initialize properties used for state when parsing tool calls in - # streaming mode + self.current_tool_id = -1 + self.current_tool_name_sent = False + self.streamed_args_for_tool: list[str] = [] self.prev_tool_call_arr: list[dict] = [] - self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: list[str] = [] # map what has been streamed for each tool so far to a list + self.bot_token = '<|python_tag|>' self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[0] self.tool_call_regex = re.compile(r'\[{.*?}\]', re.DOTALL) @@ -75,9 +73,11 @@ def extract_tool_calls_streaming( delta_text: str, delta_token_ids: Sequence[int], request: ChatCompletionRequest, + *, + stream_buffer: StreamBuffer, + **kwargs, ) -> DeltaMessage | None: - state = get_streaming_state(request) - current_text = state.current_text + current_text = stream_buffer.current_text if not (current_text.startswith(self.bot_token) or current_text.startswith('{')): return DeltaMessage(content=delta_text) diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py index d64000bc33..db82767fd8 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py @@ -16,7 +16,7 @@ FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -30,10 +30,14 @@ class Qwen2d5ToolParser(ToolParser): def __init__(self, tokenizer: object): super().__init__(tokenizer) - self.position = 0 self.tool_start_token = '' self.tool_end_token = '' self.pattern = r'(.*?)' + self.parse_cursor = 0 + self.current_tool_id = -1 + self.current_tool_name_sent = False + self.streamed_args_for_tool: list[str] = [] + self.prev_tool_call_arr: list[dict] = [] def get_argments(self, obj): if 'parameters' in obj: @@ -47,18 +51,20 @@ def extract_tool_calls_streaming( delta_text: str, delta_token_ids: Sequence[int], request: ChatCompletionRequest, + *, + stream_buffer: StreamBuffer, + **kwargs, ) -> DeltaMessage | None: - state = get_streaming_state(request) - current_text = state.current_text + current_text = stream_buffer.current_text if self.tool_start_token not in current_text: - self.position = len(current_text) + self.parse_cursor = len(current_text) return DeltaMessage(content=delta_text) # if the tool call is sended, return a empty delta message # to make sure the finish_reason will be send correctly. if self.current_tool_id > 0: return DeltaMessage(content='') - last_pos = self.position + last_pos = self.parse_cursor if self.tool_start_token not in current_text[last_pos:]: return None @@ -66,7 +72,7 @@ def extract_tool_calls_streaming( text, action = new_delta.split(self.tool_start_token) if len(text) > 0: - self.position = self.position + len(text) + self.parse_cursor = self.parse_cursor + len(text) return DeltaMessage(content=text) action = action.strip() diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py index 88cdd11a55..df2c0bfc85 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py @@ -2,9 +2,10 @@ import json import re from collections.abc import Sequence -from dataclasses import dataclass +import partial_json_parser import shortuuid +from partial_json_parser.core.options import Allow from lmdeploy.serve.openai.protocol import ( ChatCompletionRequest, @@ -15,28 +16,15 @@ FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager +from .utils import find_common_prefix, is_complete_json logger = get_logger('lmdeploy') -@dataclass -class ParserState: - """Maintains the state of parsing during tool call extraction.""" - position: int = 0 # Current position in the text being parsed - current_index: int = -1 # Index of the current tool call - parsing_reasoning: bool = False # Whether currently parsing reasoning content - - id: str = '' # ID of the current tool call - - def reset_tool_call(self): - """Called when `` finish tag occurred.""" - self.id = '' - - @ToolParserManager.register_module(['qwen', 'qwen3']) class Qwen3ToolParser(ToolParser): """Parser for Qwen3 model's tool call format. @@ -50,6 +38,12 @@ def __init__(self, tokenizer: object): self.tool_start_token = '' self.tool_end_token = '' self.tool_call_pat = re.compile(r'\n*(.*?)', re.DOTALL) + self.parse_cursor = 0 + self.qwen_tool_serial_index = -1 + self.qwen_active_tool_call_id = '' + self.current_tool_name_sent = False + self.prev_tool_call_arr: list[dict] = [] + self.streamed_args_for_tool: list[str] = [] def get_argments(self, obj): """Extract arguments from tool call object, handling different formats. @@ -62,60 +56,27 @@ def get_argments(self, obj): return obj.get('arguments') return None - def _split(self, parser_state: ParserState, parsing_content: str): + def _split(self, parsing_content: str): """Split content into tuple: (text_content, tool_content, has_tool_end) This method parses the model output and separates it into regular text, and tool call content. """ - # tool call try: start_idx = parsing_content.index(self.tool_start_token) - # move to the beginning of tool_start_token - parser_state.position += start_idx + self.parse_cursor += start_idx except ValueError: - parser_state.position += len(parsing_content) + self.parse_cursor += len(parsing_content) return parsing_content, '', False try: end_idx = parsing_content.index(self.tool_end_token) except ValueError: - # position holds until tool_end_token is found return parsing_content[:start_idx], '', False - # move position to the end of tool_end_token - parser_state.position += (end_idx - start_idx) + len(self.tool_end_token) - return parsing_content[:start_idx], parsing_content[start_idx + len(self.tool_start_token):end_idx], True - - def _parse_delta_tool_call(self, parser_state: ParserState, tool_content: str) -> DeltaToolCall | None: - """Parse tool content into a DeltaToolCall object. - - This method handles parsing tool calls only when it's a valid tool - """ - parsable_arr = tool_content.strip() - try: - tool_call_arr: dict = json.loads(parsable_arr) - except json.JSONDecodeError: - logger.debug('cannot parse into JSON yet') - return - - fcall = DeltaFunctionCall() - func_name = tool_call_arr.get('name') - if func_name: - fcall.name = func_name - args = self.get_argments(tool_call_arr) - if args and isinstance(args, dict): - fcall.arguments = json.dumps(args, ensure_ascii=False) - # Return None if no new information to send - if not fcall.name and not fcall.arguments: - return - if not parser_state.id: - # A new tool call parsed, allocate a new id & index - parser_state.id = f'chatcmpl-tool-{shortuuid.random()}' - parser_state.current_index += 1 - # Create and return the DeltaToolCall object - return DeltaToolCall( - id=parser_state.id, - index=parser_state.current_index, - function=fcall.model_dump(exclude_none=True), + self.parse_cursor += (end_idx - start_idx) + len(self.tool_end_token) + return ( + parsing_content[:start_idx], + parsing_content[start_idx + len(self.tool_start_token):end_idx], + True, ) def extract_tool_calls_streaming( @@ -123,36 +84,86 @@ def extract_tool_calls_streaming( delta_text: str, delta_token_ids: Sequence[int], request: ChatCompletionRequest, + *, + stream_buffer: StreamBuffer, + **kwargs, ) -> DeltaMessage | None: - """Extract tool calls from streaming model output. - - This method processes incremental model output to extract tool calls, reasoning content, and regular text - content in a streaming fashion. It maintains parser state between calls to handle partial outputs. - """ - state = get_streaming_state(request) - current_text = state.current_text - - parser_state = getattr(request, '_tool_parser_state', None) - if parser_state is None: - parser_state = ParserState() - setattr(request, '_tool_parser_state', parser_state) - - # Split the new content into text and tool content - split_result = self._split(parser_state, current_text[parser_state.position:]) + """Extract tool calls from streaming model output.""" + current_text = stream_buffer.current_text + split_result = self._split(current_text[self.parse_cursor:]) text_content, tool_content, has_tool_end = split_result delta = DeltaMessage() - # Add each type of content to the delta message if present if text_content: delta.content = text_content + if tool_content: - # Parse tool content into a DeltaToolCall object - delta_tool_call = self._parse_delta_tool_call(parser_state, tool_content) - if delta_tool_call is not None: - delta.tool_calls = [delta_tool_call] - if has_tool_end: - parser_state.reset_tool_call() - return delta + strip = tool_content.strip() + if strip: + flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR + obj: dict | None + try: + obj = partial_json_parser.loads(strip, flags) + except partial_json_parser.core.exceptions.MalformedJSON: + logger.debug('cannot parse into partial JSON yet') + obj = None + + if obj is not None and not self.current_tool_name_sent: + func_name = obj.get('name') + if func_name: + if not self.qwen_active_tool_call_id: + self.qwen_active_tool_call_id = f'chatcmpl-tool-{shortuuid.random()}' + self.qwen_tool_serial_index += 1 + self.streamed_args_for_tool.append('') + idx = self.qwen_tool_serial_index + delta.tool_calls = [ + DeltaToolCall( + id=self.qwen_active_tool_call_id, + index=idx, + type='function', + function=DeltaFunctionCall(name=func_name).model_dump(exclude_none=True), + ) + ] + self.current_tool_name_sent = True + self.prev_tool_call_arr = [dict(obj)] + elif obj is not None: + idx = self.qwen_tool_serial_index + args = self.get_argments(obj) + cur_arguments = args if isinstance(args, dict) else None + prev_arguments = ( + self.get_argments(self.prev_tool_call_arr[0]) if self.prev_tool_call_arr else None + ) + is_comp = is_complete_json(strip) + argument_diff = None + if cur_arguments: + cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) + if is_comp: + sent = len(self.streamed_args_for_tool[idx]) + argument_diff = cur_args_json[sent:] + elif prev_arguments: + prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) + if cur_args_json != prev_args_json: + prefix = find_common_prefix(prev_args_json, cur_args_json) + sent = len(self.streamed_args_for_tool[idx]) + argument_diff = prefix[sent:] + if argument_diff is not None: + delta.tool_calls = [ + DeltaToolCall( + index=idx, + id=self.qwen_active_tool_call_id, + function=DeltaFunctionCall( + arguments=argument_diff).model_dump(exclude_none=True), + ) + ] + self.streamed_args_for_tool[idx] += argument_diff + self.prev_tool_call_arr = [obj] + + if has_tool_end: + self.qwen_active_tool_call_id = '' + self.current_tool_name_sent = False + self.prev_tool_call_arr = [] + + return delta if delta.content is not None or delta.tool_calls else None def extract_tool_calls( self, @@ -166,19 +177,18 @@ def extract_tool_calls( """ text = model_output - # Extract tool calls (content inside tags) buf = [] scan_pos = 0 tool_calls = [] for idx, match in enumerate(self.tool_call_pat.finditer(text)): - buf.append(text[scan_pos:match.start()]) # Add text before the tag + buf.append(text[scan_pos:match.start()]) scan_pos = match.end() - action = json.loads(match.group(1)) # Parse the tool call JSON + action = json.loads(match.group(1)) name, arguments = action['name'], json.dumps(action['arguments'], ensure_ascii=False) tool_calls.append(ToolCall(function=FunctionCall(name=name, arguments=arguments))) if scan_pos < len(text): - buf.append(text[scan_pos:]) # Add remaining text - text = ''.join(buf) # Reconstruct text without tags + buf.append(text[scan_pos:]) + text = ''.join(buf) return ExtractedToolCallInformation( content=text, diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index 62e2b279f9..ebea434233 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -2,7 +2,6 @@ import json import re from collections.abc import Sequence -from dataclasses import dataclass from typing import Any import shortuuid @@ -16,7 +15,7 @@ FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -38,19 +37,6 @@ def _parse_tool_call_arguments_dict(arguments: Any) -> dict[str, Any] | None: return None -@dataclass -class ParserState: - """Maintains the state of parsing during tool call extraction.""" - position: int = 0 # Current position in the text being parsed - current_index: int = -1 # Index of the current tool call - - id: str = '' # ID of the current tool call - - def reset_tool_call(self): - """Called when `` finish tag occurred.""" - self.id = '' - - @ToolParserManager.register_module(['qwen3coder']) class Qwen3CoderToolParser(ToolParser): """Parser for Qwen3 Coder model's tool call format. @@ -70,6 +56,13 @@ def __init__(self, tokenizer: object): self.param_end_token = '' self.tool_call_pat = re.compile(r'\n*(.*?)', re.DOTALL) + self.parse_cursor = 0 + self.qwen_tool_serial_index = -1 + self.qwen_active_tool_call_id = '' + self.coder_has_emitted_name = False + self.coder_has_emitted_json_start = False + self.coder_json_closed = False + self.coder_emitted_param_names: set[str] = set() def _normalize_request_messages(self, messages: list[dict]) -> list[dict] | None: """Return a render-safe copy of request messages when needed.""" @@ -121,13 +114,13 @@ def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionReques return request return request.model_copy(update={'messages': normalized_messages}) - def _split(self, parser_state: ParserState, parsing_content: str) -> tuple[str, str, bool]: + def _split(self, parsing_content: str) -> tuple[str, str, bool]: """Split content into tuple: (text_content, tool_content, has_tool_end)""" try: start_idx = parsing_content.index(self.tool_start_token) - parser_state.position += start_idx + self.parse_cursor += start_idx except ValueError: - parser_state.position += len(parsing_content) + self.parse_cursor += len(parsing_content) return parsing_content, '', False try: @@ -136,7 +129,7 @@ def _split(self, parser_state: ParserState, parsing_content: str) -> tuple[str, return parsing_content[:start_idx], parsing_content[start_idx:], False rem = end_idx - start_idx - parser_state.position += rem + len(self.tool_end_token) + self.parse_cursor += rem + len(self.tool_end_token) return parsing_content[:start_idx], parsing_content[start_idx:end_idx + len(self.tool_end_token)], True def _extract_params(self, content: str) -> tuple[str | None, dict[str, Any], bool]: @@ -195,15 +188,13 @@ def extract_tool_calls_streaming( delta_text: str, delta_token_ids: Sequence[int], request: ChatCompletionRequest, + *, + stream_buffer: StreamBuffer, + **kwargs, ) -> DeltaMessage | None: - state = get_streaming_state(request) - current_text = state.current_text - parser_state = getattr(request, '_tool_parser_state', None) - if parser_state is None: - parser_state = ParserState() - setattr(request, '_tool_parser_state', parser_state) - - split_result = self._split(parser_state, current_text[parser_state.position:]) + current_text = stream_buffer.current_text + + split_result = self._split(current_text[self.parse_cursor:]) text_content, tool_content, has_tool_end = split_result delta = DeltaMessage() @@ -211,41 +202,41 @@ def extract_tool_calls_streaming( delta.content = text_content if tool_content: - if not parser_state.id: - parser_state.id = f'chatcmpl-tool-{shortuuid.random()}' - parser_state.current_index += 1 - parser_state.has_emitted_name = False - parser_state.has_emitted_json_start = False - parser_state.json_closed = False - parser_state.emitted_params = set() + if not self.qwen_active_tool_call_id: + self.qwen_active_tool_call_id = f'chatcmpl-tool-{shortuuid.random()}' + self.qwen_tool_serial_index += 1 + self.coder_has_emitted_name = False + self.coder_has_emitted_json_start = False + self.coder_json_closed = False + self.coder_emitted_param_names.clear() func_name, args_dict, is_func_closed = self._extract_params(tool_content) fcall_delta = DeltaFunctionCall() has_updates = False - if func_name and not getattr(parser_state, 'has_emitted_name', False): + if func_name and not self.coder_has_emitted_name: fcall_delta.name = func_name - parser_state.has_emitted_name = True + self.coder_has_emitted_name = True has_updates = True json_fragments = [] - if not getattr(parser_state, 'has_emitted_json_start', False): + if not self.coder_has_emitted_json_start: if args_dict or is_func_closed: json_fragments.append('{') - parser_state.has_emitted_json_start = True + self.coder_has_emitted_json_start = True for k, v in args_dict.items(): - if k not in parser_state.emitted_params: - prefix = ', ' if len(parser_state.emitted_params) > 0 else '' + if k not in self.coder_emitted_param_names: + prefix = ', ' if len(self.coder_emitted_param_names) > 0 else '' serialized = json.dumps(v, ensure_ascii=False) json_fragments.append(f'{prefix}\"{k}\": {serialized}') - parser_state.emitted_params.add(k) + self.coder_emitted_param_names.add(k) - if is_func_closed and not getattr(parser_state, 'json_closed', False): - if getattr(parser_state, 'has_emitted_json_start', False): + if is_func_closed and not self.coder_json_closed: + if self.coder_has_emitted_json_start: json_fragments.append('}') - parser_state.json_closed = True + self.coder_json_closed = True joined_fragments = ''.join(json_fragments) if joined_fragments: @@ -254,20 +245,18 @@ def extract_tool_calls_streaming( if has_updates: parsed_delta = DeltaToolCall( - id=parser_state.id, - index=parser_state.current_index, + id=self.qwen_active_tool_call_id, + index=self.qwen_tool_serial_index, function=fcall_delta, ) delta.tool_calls = [parsed_delta] if has_tool_end: - parser_state.reset_tool_call() - # Prepare for the next tool call - if hasattr(parser_state, 'has_emitted_name'): - delattr(parser_state, 'has_emitted_name') - delattr(parser_state, 'has_emitted_json_start') - delattr(parser_state, 'json_closed') - delattr(parser_state, 'emitted_params') + self.qwen_active_tool_call_id = '' + self.coder_has_emitted_name = False + self.coder_has_emitted_json_start = False + self.coder_json_closed = False + self.coder_emitted_param_names.clear() return delta diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index cf8f969746..d6d58e0b87 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -6,6 +6,7 @@ from mmengine import Registry from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage, ExtractedToolCallInformation +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger logger = get_logger('lmdeploy') @@ -19,12 +20,6 @@ class ToolParser: """ def __init__(self, tokenizer: object): - self.prev_tool_call_arr: list[dict] = [] - # the index of the tool call that is currently being parsed - self.current_tool_id: int = -1 - self.current_tool_name_sent: bool = False - self.streamed_args_for_tool: list[str] = [] - self.model_tokenizer = tokenizer @cached_property @@ -51,6 +46,9 @@ def extract_tool_calls_streaming( delta_text: str, delta_token_ids: Sequence[int], request: ChatCompletionRequest, + *, + stream_buffer: StreamBuffer, + **kwargs, ) -> DeltaMessage | None: """Instance method that should be implemented for extracting tool calls from an incomplete response; for use when handling tool calls and @@ -59,13 +57,13 @@ def extract_tool_calls_streaming( Args: delta_text: The new text chunk for this iteration. delta_token_ids: The new token ids for this chunk. - request: The request object; a ``StreamingParserState`` is attached - to it via ``get_streaming_state(request)`` so that previous / - current text and token ids are available. + request: The chat completion request. + stream_buffer: Cumulative decoding state (``ResponseParser`` or a test + double); use ``stream_buffer.current_text`` for the full partial output. + Tool-specific + fields live on the parser instance (one instance per request). - Has to be an instance method because it requires state - the current - tokens/diffs, but also the information about what has previously been - parsed and extracted (see constructor). + Instance method because streaming uses the shared buffer plus parser-local state. """ raise NotImplementedError('AbstractToolParser.extract_tool_calls_streaming has not been ' 'implemented!') diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py index 5061d29de3..dda4d35806 100644 --- a/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py +++ b/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py @@ -8,7 +8,7 @@ from lmdeploy.serve.openai.protocol import ChatCompletionRequest from lmdeploy.serve.openai.reasoning_parser.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import get_streaming_state +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer TRANSFORMERS_LT_5 = Version(transformers.__version__) < Version('5.0.0') @@ -70,7 +70,7 @@ def run_reasoning_stream( request: object, chunks: list[tuple[str, list[int]]], ) -> tuple[str, str]: - state = get_streaming_state(request) + state = StreamBuffer() reasoning_acc = '' content_acc = '' for delta_text, delta_ids in chunks: @@ -79,6 +79,7 @@ def run_reasoning_stream( delta_text=delta_text or '', delta_token_ids=delta_ids, request=request, + stream_buffer=state, ) if delta_msg is not None: if delta_msg.reasoning_content: diff --git a/tests/test_lmdeploy/test_harmony_gpt_oss_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_harmony_gpt_oss_parser.py similarity index 100% rename from tests/test_lmdeploy/test_harmony_gpt_oss_parser.py rename to tests/test_lmdeploy/server/reasoning_parsers/test_harmony_gpt_oss_parser.py diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py index 5c101a683d..d576db4ce3 100644 --- a/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py +++ b/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py @@ -19,7 +19,8 @@ from lmdeploy.serve.openai.protocol import ChatCompletionRequest from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager, get_streaming_state +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer # We use Qwen3-8B's tokenizer to simulate all the test cases. @@ -77,7 +78,7 @@ def run_reasoning_stream( Returns (accumulated_reasoning, accumulated_content). """ - state = get_streaming_state(request) + state = StreamBuffer() reasoning_acc = '' content_acc = '' for delta_text, delta_ids in chunks: @@ -86,6 +87,7 @@ def run_reasoning_stream( delta_text=delta_text or '', delta_token_ids=delta_ids, request=request, + stream_buffer=state, ) if delta_msg is not None: if delta_msg.reasoning_content: diff --git a/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py b/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py new file mode 100644 index 0000000000..b74b7ab75c --- /dev/null +++ b/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py @@ -0,0 +1,441 @@ +import json +import time +from collections.abc import Generator + +import pytest +import shortuuid + +from lmdeploy.serve.openai.protocol import ( + ChatCompletionRequest, + ChatCompletionResponse, + ChatCompletionResponseChoice, + ChatCompletionResponseStreamChoice, + ChatCompletionStreamResponse, + ChatMessage, + DeltaMessage, + UsageInfo, +) +from lmdeploy.serve.openai.reasoning_parser import QwenReasoningParser +from lmdeploy.serve.openai.response_parser import StreamBuffer +from lmdeploy.serve.openai.tool_parser import Qwen3ToolParser +from lmdeploy.tokenizer import Tokenizer + + +@pytest.fixture(scope='module') +def tokenizer(): + from lmdeploy.tokenizer import HuggingFaceTokenizer + return HuggingFaceTokenizer('Qwen/Qwen3-8B') + +@pytest.fixture() +def reasoning_parser(tokenizer): + return QwenReasoningParser(tokenizer) + +@pytest.fixture() +def tool_parser(tokenizer): + return Qwen3ToolParser(tokenizer) + +DELTA_TEXT_SEQUENCE = [ + # (delta_text, reasoning_content, content, tool_calls) + ('', None, None, []), + ('\n', '\n', None, []), + ('好的', '好的', None, []), + ('，', '，', None, []), + ('用户', '用户', None, []), + ('问', '问', None, []), + ('的是', '的是', None, []), + ('北京', '北京', None, []), + ('的', '的', None, []), + ('天气', '天气', None, []), + ('怎么样', '怎么样', None, []), + ('。', '。', None, []), + ('我', '我', None, []), + ('需要', '需要', None, []), + ('调', '调', None, []), + ('用', '用', None, []), + ('get', 'get', None, []), + ('_weather', '_weather', None, []), + ('这个', '这个', None, []), + ('工具', '工具', None, []), + ('来', '来', None, []), + ('获取', '获取', None, []), + ('信息', '信息', None, []), + ('。', '。', None, []), + ('首先', '首先', None, []), + ('，', '，', None, []), + ('确认', '确认', None, []), + ('用户', '用户', None, []), + ('提供的', '提供的', None, []), + ('地点', '地点', None, []), + ('是', '是', None, []), + ('北京', '北京', None, []), + ('，', '，', None, []), + ('参数', '参数', None, []), + ('正确', '正确', None, []), + ('。', '。', None, []), + ('然后', '然后', None, []), + ('检查', '检查', None, []), + ('工具', '工具', None, []), + ('的', '的', None, []), + ('参数', '参数', None, []), + ('要求', '要求', None, []), + ('，', '，', None, []), + ('只需要', '只需要', None, []), + ('location', 'location', None, []), + ('，', '，', None, []), + ('类型', '类型', None, []), + ('是', '是', None, []), + ('字符串', '字符串', None, []), + ('。', '。', None, []), + ('于是', '于是', None, []), + ('构造', '构造', None, []), + ('参数', '参数', None, []), + ('对象', '对象', None, []), + ('，', '，', None, []), + ('调', '调', None, []), + ('用', '用', None, []), + ('函数', '函数', None, []), + ('，', '，', None, []), + ('返回', '返回', None, []), + ('结果', '结果', None, []), + ('。', '。', None, []), + ('确保', '确保', None, []), + ('没有', '没有', None, []), + ('遗漏', '遗漏', None, []), + ('必要', '必要', None, []), + ('参数', '参数', None, []), + ('，', '，', None, []), + ('比如', '比如', None, []), + ('location', 'location', None, []), + ('是', '是', None, []), + ('必须', '必须', None, []), + ('的', '的', None, []), + ('，', '，', None, []), + ('这里', '这里', None, []), + ('已经', '已经', None, []), + ('提供', '提供', None, []), + ('，', '，', None, []), + ('所以', '所以', None, []), + ('没问题', '没问题', None, []), + ('。', '。', None, []), + ('最后', '最后', None, []), + ('将', '将', None, []), + ('结果', '结果', None, []), + ('以', '以', None, []), + ('自然', '自然', None, []), + ('语言', '语言', None, []), + ('回复', '回复', None, []), + ('用户', '用户', None, []), + ('。\n', '。\n', None, []), + ('', None, None, []), + ('\n\n', None, '\n\n', []), + ('', None, None, []), + ('\n', None, None, '\n'), + ('{"', None, None, '{"'), + ('name', None, None, 'name'), + ('":', None, None, '":'), + (' "', None, None, ' "'), + ('get', None, None, 'get'), + ('_weather', None, None, '_weather'), + ('",', None, None, '",'), + (' "', None, None, ' "'), + ('arguments', None, None, 'arguments'), + ('":', None, None, '":'), + (' {"', None, None, ' {"'), + ('location', None, None, 'location'), + ('":', None, None, '":'), + (' "', None, None, ' "'), + ('北京', None, None, '北京'), + ('"}}\n', None, None, '"}}\n'), + ('', None, None, None) +] + +DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS = DELTA_TEXT_SEQUENCE + [ + '\n\n', + '', + '\n', + '{"', + 'name', + '":', + ' "', + 'get', + '_weather', + '",', + ' "', + 'arguments', + '":', + ' {"', + 'location', + '":', + ' "', + '上海', + '"}}\n', + '', +] + +EXPECTED_CONTENT = '' +EXPECTED_REASONING_CONTENT = ''.join(( + '好的，用户问的是北京的天气怎么样。我需要调用get_weather这个工具来获取信息。', + '首先，确认用户提供的地点是北京，参数正确。然后检查工具的参数要求，', + '只需要location，类型是字符串。于是构造参数对象，调用函数，返回结果。', + '确保没有遗漏必要参数，比如location是必须的，这里已经提供，所以没问题。', + '最后将结果以自然语言回复用户。', +)) + + +def _normalize_delta_sequence(text_sequence: list) -> list[str]: + """Flatten streaming fixtures that use (delta, ...) tuples (possibly mixed + with str chunks).""" + if not text_sequence: + return [] + out = [] + for item in text_sequence: + out.append(item[0] if isinstance(item, tuple) else item) + return out + + +def _chat_completion_v1( + tokenizer: Tokenizer, + reasoning_parser: QwenReasoningParser, + tool_parser: Qwen3ToolParser, + request: ChatCompletionRequest, + text_sequence: list[str]) -> ChatCompletionResponse | Generator[ChatCompletionStreamResponse, None, None]: + request_id = f'chat-{shortuuid.random()}' + created_time = int(time.time()) + model_name = request.model + delta_chunks = _normalize_delta_sequence(text_sequence) + if request.stream: + parser_state = StreamBuffer() + has_parser = tool_parser is not None or reasoning_parser is not None + + def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, None, None]: + finish_reason = 'stop' + for text in delta_chunks: + print(f'delta_text: {text}') + # delta_message = DeltaMessage(role='assistant', content=None) + delta_message = DeltaMessage(role='assistant', content=text) if not has_parser else None + content = text + delta_token_ids = tokenizer.encode(content, add_bos=False) + parser_state.update(content, delta_token_ids) + if request.tool_choice != 'none' and tool_parser is not None: + delta_message = DeltaMessage(role='assistant') + tool_delta = tool_parser.extract_tool_calls_streaming( + delta_text=content, + delta_token_ids=delta_token_ids, + request=request, + stream_buffer=parser_state, + ) + print(f'tool_delta: {tool_delta}') + if tool_delta is not None: + delta_message.tool_calls = tool_delta.tool_calls + delta_message.content = tool_delta.content + if reasoning_parser is not None: + if tool_parser is None or delta_message is None: + content = text + elif delta_message.content is not None: + # delta_message.content is `content` if there is no tool call information in it + content = delta_message.content + # There might be reasoning content in `delta_message.content`. + # So we set it to None and let reasoning parser to extract the reasoning and content. + delta_message.content = None + else: + # tool_parser is consuming tool call information. We set Nont content to jump + # parsing reasoning. + content = None + reasoning_delta = reasoning_parser.extract_reasoning_streaming( + delta_text=content, + delta_token_ids=delta_token_ids, + request=request, + stream_buffer=parser_state, + ) + print(f'reasoning_delta: {reasoning_delta}') + if reasoning_delta is not None: + delta_message.reasoning_content = reasoning_delta.reasoning_content + delta_message.content = reasoning_delta.content + parser_state.step() + choice_data = ChatCompletionResponseStreamChoice(index=0, + delta=delta_message, + finish_reason=finish_reason) + response = ChatCompletionStreamResponse( + id=request_id, + created=created_time, + model=model_name, + choices=[choice_data] + ) + yield response + + return completion_stream_generator() + + # copied and simplified from api_server.py:chat_completions_v1 + text = ''.join(delta_chunks) + tool_calls = None + reasoning_content = None + finish_reason = 'stop' + if request.tool_choice != 'none' and tool_parser is not None: + tool_call_info = tool_parser.extract_tool_calls(text, request=request) + text, tool_calls = tool_call_info.content, tool_call_info.tool_calls + if isinstance(tool_calls, list) and len(tool_calls): + if finish_reason == 'stop': + finish_reason = 'tool_calls' + + if reasoning_parser is not None: + reasoning_content, text = reasoning_parser.extract_reasoning(text, request) + + choices = [] + choice_data = ChatCompletionResponseChoice( + index=0, + message=ChatMessage(role='assistant', content=text, tool_calls=tool_calls, reasoning_content=reasoning_content), + finish_reason=finish_reason, + ) + choices.append(choice_data) + + return ChatCompletionResponse( + id=request_id, + created=created_time, + model=model_name, + choices=choices, + usage=UsageInfo(), + ) + + +# def _stream_parse( +# tokenizer: Tokenizer, +# reasoning_parser: QwenReasoningParser, +# tool_parser: Qwen3ToolParser, +# request: ChatCompletionRequest, +# text_sequence: list[str], +# ) -> tuple[str, str, list[DeltaToolCall]]: +# # Call parser.extract_tool_calls_streaming with delta_text specified in `DELTA_TEXT_SEQUENCE`. +# # `current_text` and `previous_text` init values and update logic +# # can be found in lmdeploy/serve/openai/api_server.py:455-523. +# content = '' +# reasoning_content = '' +# tool_calls = {} + +# for stream_resp in _chat_completion_v1(tokenizer, reasoning_parser, tool_parser, request, text_sequence): +# delta_message: DeltaMessage = stream_resp.choices[0].delta +# if delta_message.content: +# content += delta_message.content +# if delta_message.reasoning_content: +# reasoning_content += delta_message.reasoning_content +# if delta_message.tool_calls: +# for c in delta_message.tool_calls: +# existing_call = tool_calls.get(c.id, None) +# if not existing_call: +# tool_calls[c.id] = c +# continue +# # merge with existing +# if c.function.name: +# existing_call.function.name = c.function.name +# if c.function.arguments: +# existing_call.function.arguments = existing_call.function.arguments or '' +# existing_call.function.arguments += c.function.arguments +# return content, reasoning_content, list(sorted(tool_calls.values(), key=lambda x: x.index)) + + + +class TestQwen3ToolStreamingParser: + """Tests for Qwen3ToolParser streaming mode.""" + + @pytest.mark.parametrize('text_sequence', [DELTA_TEXT_SEQUENCE]) + def test_parser_stream(self, tokenizer, reasoning_parser, tool_parser, + text_sequence: list[tuple[str, str, str, str]]): + """Test streaming parser with single and multiple tool calls.""" + request = ChatCompletionRequest(model='qwen', messages=[], stream=True) + delta_texts = [t[0] for t in text_sequence] + responses = _chat_completion_v1(tokenizer, reasoning_parser, tool_parser, request, delta_texts) + for response, t in zip(responses, text_sequence): + delta_message: DeltaMessage = response.choices[0].delta + print(f'delta_message: {delta_message}') + assert delta_message.reasoning_content == t[1] + assert delta_message.content == t[2] + # assert delta_message.tool_calls == t[3] + + + def test_incomplete_tool_call_streaming(self, tokenizer, reasoning_parser, tool_parser): + """Test streaming parser with incomplete tool call (missing end + tag).""" + request = ChatCompletionRequest(model='qwen', messages=[], stream=True) + + # Incomplete tool call without end tag + text_sequence = ['好的', '，', '让我', '调用', '工具', '。', 'Вот', '\n', 'ذهب', '\n', + '{"name": "get_weather", "arguments": {"location": "北京"'] + responses = _chat_completion_v1( + tokenizer, reasoning_parser, tool_parser, request, text_sequence) + for response in responses: + delta_message: DeltaMessage = response.choices[0].delta + print(f'delta_message: {delta_message}') + assert delta_message.tool_calls is None + # Should not parse tool call since it's incomplete + + +class TestQwen3ToolNonStreamingParser: + """Tests for Qwen3ToolParser non-streaming mode.""" + + @pytest.mark.parametrize('text_sequence', [DELTA_TEXT_SEQUENCE, DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS]) + def test_parser_nonstream(self, tokenizer, reasoning_parser, tool_parser, text_sequence: list[str]): + """Test non-streaming parser with single and multiple tool calls.""" + full = ''.join(_normalize_delta_sequence(text_sequence)) + req = ChatCompletionRequest(model='qwen', messages=[], stream=False) + tool_ref = tool_parser.extract_tool_calls(full, request=req) + + resp: ChatCompletionResponse = _chat_completion_v1( + tokenizer, reasoning_parser, tool_parser, req, text_sequence) + + assert len(resp.choices) == 1 + first_message = resp.choices[0].message + assert (first_message.content or '').strip() == EXPECTED_CONTENT + assert (first_message.reasoning_content or '').strip() == EXPECTED_REASONING_CONTENT + assert len(first_message.tool_calls) == len(tool_ref.tool_calls) + for parsed_call, ref_call in zip(first_message.tool_calls, tool_ref.tool_calls): + assert parsed_call.function.name == ref_call.function.name + assert json.loads(parsed_call.function.arguments) == json.loads(ref_call.function.arguments) + + def test_no_think_nonstream(self, tokenizer, reasoning_parser, tool_parser): + """Test non-streaming parser with plain text (no thinking tags).""" + text_sequence = [ + '你好', + '呀', + '！', + '✨', + '', + ' 很', + '高兴', + '见到', + '你', + '！', + ] + resp: ChatCompletionResponse = _chat_completion_v1( + tokenizer, reasoning_parser, tool_parser, + ChatCompletionRequest(model='qwen', messages=[], stream=False), + text_sequence) + + assert len(resp.choices) == 1 + first_message = resp.choices[0].message + assert first_message.content == '你好呀！✨ 很高兴见到你！' + assert first_message.reasoning_content is None + + def test_invalid_json_tool_call(self, tokenizer, reasoning_parser, tool_parser): + """Test non-streaming parser with invalid JSON in tool call.""" + # Invalid JSON in tool call + text_sequence = ['好的，让我调用工具。', 'Вот', '\n', 'ذهب', '\n', + '{"name": "get_weather", "arguments": {invalid json}}', '666', '\n'] + + resp: ChatCompletionResponse = _chat_completion_v1( + tokenizer, reasoning_parser, tool_parser, + ChatCompletionRequest(model='qwen', messages=[], stream=False), + text_sequence) + + # Should handle gracefully - tool call may not be parsed due to invalid JSON + assert len(resp.choices) == 1 + + def test_empty_tool_call_content(self, tokenizer, reasoning_parser, tool_parser): + """Test non-streaming parser with empty tool call content.""" + # Empty tool call + text_sequence = ['好的', '。', 'Вот', '\n', 'ذهب', '\n', '666', '\n'] + + resp: ChatCompletionResponse = _chat_completion_v1( + tokenizer, reasoning_parser, tool_parser, + ChatCompletionRequest(model='qwen', messages=[], stream=False), + text_sequence) + + assert len(resp.choices) == 1 diff --git a/tests/test_lmdeploy/test_qwen3coder_parser.py b/tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py similarity index 94% rename from tests/test_lmdeploy/test_qwen3coder_parser.py rename to tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py index d9bdacff9a..6061dee8dc 100644 --- a/tests/test_lmdeploy/test_qwen3coder_parser.py +++ b/tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py @@ -19,6 +19,7 @@ DeltaToolCall, UsageInfo, ) +from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.serve.openai.tool_parser.qwen3coder_tool_parser import Qwen3CoderToolParser TestExpects = collections.namedtuple('TestExpects', 'func_name kwargs') @@ -71,41 +72,38 @@ def _chat_completion_v1( if request.stream: def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, None, None]: - previous_text = '' - current_text = '' finish_reason = 'stop' + parser_state = StreamBuffer() has_parser = (VariableInterface.tool_parser is not None or VariableInterface.reasoning_parser is not None) for text in text_sequence: logprobs, usage = None, None delta_message = DeltaMessage(role='assistant', content=text) if has_parser: - current_text = current_text + text + parser_state.update(text, []) has_tool = VariableInterface.tool_parser is not None if request.tool_choice != 'none' and has_tool: tool_delta = VariableInterface.tool_parser.extract_tool_calls_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_message.content, - previous_token_ids=[], - current_token_ids=[], + delta_text=text, delta_token_ids=[], - request=request) + request=request, + stream_buffer=parser_state, + ) if tool_delta is not None: delta_message.tool_calls = tool_delta.tool_calls delta_message.content = tool_delta.content or '' if VariableInterface.reasoning_parser is not None: parser = VariableInterface.reasoning_parser - reasoning_delta = parser.extract_reasoning_streaming(previous_text=previous_text, - current_text=current_text, - delta_text=delta_message.content, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[]) + reasoning_delta = parser.extract_reasoning_streaming( + delta_text=delta_message.content, + delta_token_ids=[], + request=request, + stream_buffer=parser_state, + ) if reasoning_delta is not None: delta_message.reasoning_content = (reasoning_delta.reasoning_content) delta_message.content = reasoning_delta.content or '' if has_parser: - previous_text = current_text + parser_state.step() choice_data = ChatCompletionResponseStreamChoice(index=0, delta=delta_message, diff --git a/tests/test_lmdeploy/test_qwen3_parser.py b/tests/test_lmdeploy/test_qwen3_parser.py deleted file mode 100644 index ec65855e00..0000000000 --- a/tests/test_lmdeploy/test_qwen3_parser.py +++ /dev/null @@ -1,368 +0,0 @@ -import collections -import json -import time -from collections.abc import Generator - -import pytest -import shortuuid -from lmdeploy.serve.openai.reasoning_parser.qwen_qwq_reasoning_parser import QwenReasoningParser -from lmdeploy.serve.openai.tool_parser.qwen3_parser import Qwen3ToolParser - -from lmdeploy.serve.openai.api_server import VariableInterface -from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, - ChatMessage, - DeltaMessage, - DeltaToolCall, - UsageInfo, -) - -TestExpects = collections.namedtuple('TestExpects', 'func_name location') - - -class DummyTokenizer: - - def decode(self, token_ids: list[int]) -> str: - return ' '.join(map(str, token_ids)) - - def encode(self, text: str) -> list[int]: - return [ord(c) for c in text] - - -DELTA_TEXT_SEQUENCE = [ - '', - '\n', - '好的', - '，', - '用户', - '问', - '的是', - '北京', - '的', - '天气', - '怎么样', - '。', - '我', - '需要', - '调', - '用', - 'get', - '_weather', - '这个', - '工具', - '来', - '获取', - '信息', - '。', - '首先', - '，', - '确认', - '用户', - '提供的', - '地点', - '是', - '北京', - '，', - '参数', - '正确', - '。', - '然后', - '检查', - '工具', - '的', - '参数', - '要求', - '，', - '只需要', - 'location', - '，', - '类型', - '是', - '字符串', - '。', - '于是', - '构造', - '参数', - '对象', - '，', - '调', - '用', - '函数', - '，', - '返回', - '结果', - '。', - '确保', - '没有', - '遗漏', - '必要', - '参数', - '，', - '比如', - 'location', - '是', - '必须', - '的', - '，', - '这里', - '已经', - '提供', - '，', - '所以', - '没问题', - '。', - '最后', - '将', - '结果', - '以', - '自然', - '语言', - '回复', - '用户', - '。\n', - '', - '\n\n', - '', - '\n', - '{"', - 'name', - '":', - ' "', - 'get', - '_weather', - '",', - ' "', - 'arguments', - '":', - ' {"', - 'location', - '":', - ' "', - '北京', - '"}}\n', - '', -] - -DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS = DELTA_TEXT_SEQUENCE + [ - '\n\n', - '', - '\n', - '{"', - 'name', - '":', - ' "', - 'get', - '_weather', - '",', - ' "', - 'arguments', - '":', - ' {"', - 'location', - '":', - ' "', - '上海', - '"}}\n', - '', -] - -EXPECTED_CONTENT = '' -EXPECTED_REASONING_CONTENT = ''.join(( - '好的，用户问的是北京的天气怎么样。我需要调用get_weather这个工具来获取信息。', - '首先，确认用户提供的地点是北京，参数正确。然后检查工具的参数要求，', - '只需要location，类型是字符串。于是构造参数对象，调用函数，返回结果。', - '确保没有遗漏必要参数，比如location是必须的，这里已经提供，所以没问题。', - '最后将结果以自然语言回复用户。', -)) - - -def _chat_completion_v1( - request: ChatCompletionRequest, - text_sequence: list[str]) -> ChatCompletionResponse | Generator[ChatCompletionStreamResponse, None, None]: - request_id = f'chat-{shortuuid.random()}' - created_time = int(time.time()) - model_name = request.model - if request.stream: - - def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, None, None]: - previous_text = '' - current_text = '' - finish_reason = 'stop' - has_parser = VariableInterface.tool_parser is not None or VariableInterface.reasoning_parser is not None - for text in text_sequence: - logprobs, usage = None, None - delta_message = DeltaMessage(role='assistant', content=text) - if has_parser: - current_text = current_text + text - if request.tool_choice != 'none' and VariableInterface.tool_parser is not None: - tool_delta = VariableInterface.tool_parser.extract_tool_calls_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_message.content, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[], - request=request) - if tool_delta is not None: - delta_message.tool_calls = tool_delta.tool_calls - delta_message.content = tool_delta.content or '' - if VariableInterface.reasoning_parser is not None: - reasoning_delta = VariableInterface.reasoning_parser.extract_reasoning_streaming( - previous_text=previous_text, - current_text=current_text, - delta_text=delta_message.content, - previous_token_ids=[], - current_token_ids=[], - delta_token_ids=[]) - if reasoning_delta is not None: - delta_message.reasoning_content = reasoning_delta.reasoning_content - delta_message.content = reasoning_delta.content or '' - if has_parser: - previous_text = current_text - - choice_data = ChatCompletionResponseStreamChoice(index=0, - delta=delta_message, - finish_reason=finish_reason, - logprobs=logprobs) - response = ChatCompletionStreamResponse( - id=request_id, - created=created_time, - model=model_name, - choices=[choice_data], - usage=usage, - ) - yield response - - return completion_stream_generator() - - # copied and simplified from api_server.py:chat_completions_v1 - text = ''.join(text_sequence) - tool_calls = None - reasoning_content = None - finish_reason = 'stop' - if request.tool_choice != 'none' and VariableInterface.tool_parser is not None: - tool_call_info = VariableInterface.tool_parser.extract_tool_calls(text, request=request) - text, tool_calls = tool_call_info.content, tool_call_info.tool_calls - if isinstance(tool_calls, list) and len(tool_calls): - if finish_reason == 'stop': - finish_reason = 'tool_calls' - - if VariableInterface.reasoning_parser is not None: - reasoning_content, text = VariableInterface.reasoning_parser.extract_reasoning(text, request) - - choices = [] - choice_data = ChatCompletionResponseChoice( - index=0, - message=ChatMessage(role='assistant', content=text, tool_calls=tool_calls, reasoning_content=reasoning_content), - finish_reason=finish_reason, - ) - choices.append(choice_data) - - return ChatCompletionResponse( - id=request_id, - created=created_time, - model=model_name, - choices=choices, - usage=UsageInfo(), - ) - - -def _stream_parse(request: ChatCompletionRequest, text_sequence: list[str]) -> tuple[str, str, list[DeltaToolCall]]: - # Call parser.extract_tool_calls_streaming with delta_text specified in `DELTA_TEXT_SEQUENCE`. - # `current_text` and `previous_text` init values and update logic - # can be found in lmdeploy/serve/openai/api_server.py:455-523. - content = '' - reasoning_content = '' - tool_calls = {} - - for stream_resp in _chat_completion_v1(request, text_sequence): - delta_message: DeltaMessage = stream_resp.choices[0].delta - if delta_message.content: - content += delta_message.content - if delta_message.reasoning_content: - reasoning_content += delta_message.reasoning_content - if delta_message.tool_calls: - for c in delta_message.tool_calls: - existing_call = tool_calls.get(c.id, None) - if not existing_call: - tool_calls[c.id] = c - continue - # merge with existing - if c.function.name: - existing_call.function.name = c.function.name - if c.function.arguments: - existing_call.function.arguments = existing_call.function.arguments or '' - existing_call.function.arguments += c.function.arguments - return content, reasoning_content, list(sorted(tool_calls.values(), key=lambda x: x.index)) - - -@pytest.mark.parametrize(('text_sequence', 'expects'), [ - (DELTA_TEXT_SEQUENCE, [TestExpects('get_weather', '北京')]), - (DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS, [TestExpects('get_weather', '北京'), - TestExpects('get_weather', '上海')]), -]) -def test_parser_stream(text_sequence: list[str], expects: list[TestExpects]): - tokenizer = DummyTokenizer() - VariableInterface.tool_parser = Qwen3ToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = QwenReasoningParser(tokenizer=tokenizer) - request = ChatCompletionRequest(model='qwen', messages=[], stream=True) - content, reasoning_content, tool_calls = _stream_parse(request, text_sequence) - assert len(tool_calls) == len(expects) - for parsed_call, expected_call in zip(tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args['location'] == expected_call.location - assert content.strip() == EXPECTED_CONTENT - assert reasoning_content.strip() == EXPECTED_REASONING_CONTENT - - -@pytest.mark.parametrize(('text_sequence', 'expects'), [ - (DELTA_TEXT_SEQUENCE, [TestExpects('get_weather', '北京')]), - (DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS, [TestExpects('get_weather', '北京'), - TestExpects('get_weather', '上海')]), -]) -def test_parser_nonstream(text_sequence: list[str], expects: list[TestExpects]): - tokenizer = DummyTokenizer() - VariableInterface.tool_parser = Qwen3ToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = QwenReasoningParser(tokenizer=tokenizer) - resp: ChatCompletionResponse = _chat_completion_v1(ChatCompletionRequest(model='qwen', messages=[], stream=False), - text_sequence) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert first_message.content is None - assert first_message.reasoning_content == EXPECTED_REASONING_CONTENT - assert len(first_message.tool_calls) == len(expects) - for parsed_call, expected_call in zip(first_message.tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args['location'] == expected_call.location - - -def test_no_think_nonstream(): - text_sequence = [ - '你好', - '呀', - '！', - '✨', - '', - ' 很', - '高兴', - '见到', - '你', - '！', - ] - tokenizer = DummyTokenizer() - VariableInterface.tool_parser = Qwen3ToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = QwenReasoningParser(tokenizer=tokenizer) - resp: ChatCompletionResponse = _chat_completion_v1(ChatCompletionRequest(model='qwen', messages=[], stream=False), - text_sequence) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert first_message.content == '你好呀！✨ 很高兴见到你！' - assert first_message.reasoning_content is None From bc0502e6d6cdb965a86779c58e72e8b25c34b374 Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 04:29:11 +0000 Subject: [PATCH 07/14] agent's 2nd refactor version --- lmdeploy/serve/openai/api_server.py | 218 +++++------------- lmdeploy/serve/openai/protocol.py | 4 +- .../gpt_oss_reasoning_parser.py | 5 +- .../reasoning_parser/qwen_reasoning_parser.py | 1 - lmdeploy/serve/openai/response_parser.py | 126 +++++++--- .../tool_parser/internlm2_tool_parser.py | 23 ++ .../openai/tool_parser/llama3_tool_parser.py | 20 ++ .../openai/tool_parser/qwen2d5_tool_parser.py | 17 ++ .../openai/tool_parser/qwen3_tool_parser.py | 27 ++- .../tool_parser/qwen3coder_tool_parser.py | 17 ++ .../serve/openai/tool_parser/tool_parser.py | 24 ++ .../server/parsers/test_qwen3_5_parsers.py | 179 ++++++++++++++ .../server/parsers/test_qwen_parsers.py | 208 +++++++++++++++++ .../server/tool_parsers/test_qwen3_parser.py | 2 +- 14 files changed, 663 insertions(+), 208 deletions(-) create mode 100644 tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py create mode 100644 tests/test_lmdeploy/server/parsers/test_qwen_parsers.py diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index cca5111e06..97d38c95b9 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -1,7 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. # yapf: disable import asyncio -import copy import json import os import re @@ -10,7 +9,10 @@ from contextlib import asynccontextmanager from functools import partial from http import HTTPStatus -from typing import Literal +from typing import TYPE_CHECKING, Literal + +if TYPE_CHECKING: + from transformers import PreTrainedTokenizerBase import uvicorn from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request, status @@ -73,12 +75,10 @@ UpdateParamsRequest, UsageInfo, ) -from lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser import GptOssReasoningParser from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager from lmdeploy.serve.openai.response_parser import ResponseParser from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParserManager from lmdeploy.serve.utils.server_utils import validate_json_request -from lmdeploy.tokenizer import DetokenizeState, Tokenizer from lmdeploy.utils import get_logger # yapf: enable @@ -177,72 +177,13 @@ def always_success(req, server_context): return None -def _create_completion_logprobs(tokenizer: Tokenizer, - token_ids: list[int] | None = None, - logprobs: list[dict[int, float]] | None = None, - skip_special_tokens: bool = True, - offset: int = 0, - all_token_ids: list[int] | None = None, - state: DetokenizeState = None, - spaces_between_special_tokens: bool = True): - """Create openai LogProbs for completion. - - Args: - tokenizer (Tokenizer): tokenizer. - token_ids (list[int]): output token ids. - logprobs (list[dict[int, float]]): the top logprobs for each output - position. - skip_special_tokens (bool): Whether or not to remove special tokens - in the decoding. Default to be True. - offset (int): text offset. - all_token_ids (int): the history output token ids. - state (DetokenizeState): tokenizer decode state. - spaces_between_special_tokens (bool): Whether or not to add spaces - around special tokens. The behavior of Fast tokenizers is to have - this to False. This is setup to True in slow tokenizers. - """ - if logprobs is None or len(logprobs) == 0: - return None, None, None, None - - if all_token_ids is None: - all_token_ids = [] - if state is None: - state = DetokenizeState() - - out_logprobs = LogProbs() - out_logprobs.top_logprobs = [] - for token_id, tops in zip(token_ids, logprobs): - out_logprobs.text_offset.append(offset) - out_logprobs.token_logprobs.append(tops[token_id]) - - res = {} - out_state = None - for top_id, prob in tops.items(): - response, _state = tokenizer.detokenize_incrementally( - all_token_ids + [top_id], - copy.deepcopy(state), - skip_special_tokens=skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens) - res[response] = prob - if top_id == token_id: - out_state = _state - offset += len(response) - out_logprobs.tokens.append(response) - - out_logprobs.top_logprobs.append(res) - state = out_state - all_token_ids.append(token_id) - - return out_logprobs, offset, all_token_ids, state - - -def _create_chat_completion_logprobs(tokenizer: Tokenizer, +def _create_chat_completion_logprobs(tokenizer: 'PreTrainedTokenizerBase', token_ids: list[int] | None = None, logprobs: list[dict[int, float]] | None = None): """Create openai LogProbs for chat.completion. Args: - tokenizer (Tokenizer): tokenizer. + tokenizer (PreTrainedTokenizerBase): tokenizer. token_ids (list[int]): output token ids. logprobs (list[dict[int, float]]): the top logprobs for each output position. @@ -256,7 +197,7 @@ def _create_chat_completion_logprobs(tokenizer: Tokenizer, for token_id, tops in zip(token_ids, logprobs): item = ChatCompletionTokenLogprob(token='', bytes=[], logprob=0.0, top_logprobs=[]) for top_id, prob in tops.items(): - token = tokenizer.model.model.convert_ids_to_tokens(top_id) + token = tokenizer.convert_ids_to_tokens(top_id) if isinstance(token, bytes): _bytes = list(token) token = token.decode('utf-8', errors='backslashreplace') @@ -292,7 +233,8 @@ async def terminate(): # modified from https://github.com/vllm-project/vllm/blob/v0.5.4/vllm/entrypoints/openai/logits_processors.py#L51 # noqa -def logit_bias_logits_processor(logit_bias: dict[int, float] | dict[str, float], tokenizer) -> LogitsProcessor: +def logit_bias_logits_processor(logit_bias: dict[int, float] | dict[str, float], + tokenizer: 'PreTrainedTokenizerBase') -> LogitsProcessor: try: # Convert token_id to integer # Clamp the bias between -100 and 100 per OpenAI API spec @@ -425,23 +367,10 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque if isinstance(request.stop, str): request.stop = [request.stop] - tokenizer = VariableInterface.async_engine.tokenizer.model - response_parser = ResponseParser(request=request, tokenizer=tokenizer) - - # Harmony GPT-OSS: explicit `--reasoning-parser gpt-oss`, or GptOssForCausalLM arch. - gpt_oss_parser = None - if isinstance(response_parser.reasoning_parser, GptOssReasoningParser): - gpt_oss_parser = response_parser.reasoning_parser - elif VariableInterface.async_engine.arch == 'GptOssForCausalLM': - gpt_oss_parser = GptOssReasoningParser(tokenizer, **response_parser._kwargs) - + tokenizer = VariableInterface.async_engine.tokenizer.model.model gen_logprobs, logits_processors = None, None if request.logprobs and request.top_logprobs: gen_logprobs = request.top_logprobs - response_format = None - if request.response_format and request.response_format.type != 'text': - response_format = request.response_format.model_dump() - if request.logit_bias is not None: try: logits_processors = [ @@ -452,7 +381,9 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque random_seed = request.seed if request.seed else None max_new_tokens = (request.max_completion_tokens if request.max_completion_tokens else request.max_tokens) - + response_format = None + if request.response_format and request.response_format.type != 'text': + response_format = request.response_format.model_dump() gen_config = GenerationConfig( max_new_tokens=max_new_tokens, do_sample=True, @@ -475,27 +406,10 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque with_cache=with_cache, preserve_cache=preserve_cache, ) + response_parser = ResponseParser(request=request, tokenizer=tokenizer) + # request might be adjusted by tool parser + request = response_parser.request - tools = None - if request.tools and request.tool_choice != 'none': - gen_config.skip_special_tokens = False - # internlm2 only uses contents inside function regardless of 'type' - if not isinstance(request.tool_choice, str): - if gpt_oss_parser: - tools = [ - item.model_dump() for item in request.tools - if item.function.name == request.tool_choice.function.name - ] - else: - tools = [ - item.function.model_dump() for item in request.tools - if item.function.name == request.tool_choice.function.name - ] - else: - if gpt_oss_parser: - tools = [item.model_dump() for item in request.tools] - else: - tools = [item.function.model_dump() for item in request.tools] # text completion for string input do_preprocess = False if isinstance(request.messages, str) else request.do_preprocess chat_template_kwargs = request.chat_template_kwargs or {} @@ -511,7 +425,7 @@ async def chat_completions_v1(request: ChatCompletionRequest, raw_request: Reque request.messages, session, gen_config=gen_config, - tools=tools, + tools=request.tools, reasoning_effort=request.reasoning_effort, stream_response=True, # always use stream to enable batching sequence_start=True, @@ -556,30 +470,21 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: completion_tokens=res.generate_token_len, total_tokens=total_tokens, ) - + print(f'[completion_stream_generator] res.response: {res.response}, res.token_ids: {res.token_ids}') delta_token_ids = res.token_ids if res.token_ids is not None else [] - if gpt_oss_parser: - delta_message = gpt_oss_parser.parse_streaming(res.token_ids) - if res.finish_reason == 'stop' and len(delta_message.tool_calls) > 0: + delta_message, tool_emitted = response_parser.stream_chunk( + res.response, + delta_token_ids + ) + if tool_emitted: + streaming_tools = True + + if (request.tool_choice != 'none' and response_parser.tool_parser is not None): + if res.finish_reason == 'stop' and streaming_tools is True: res.finish_reason = 'tool_calls' - else: - if response_parser is not None: - delta_message, tool_emitted = response_parser.stream_chunk( - res.response, - delta_token_ids - ) - if tool_emitted: - streaming_tools = True - else: - delta_message = DeltaMessage(role='assistant', content=res.response) - - if (request.tool_choice != 'none' and response_parser is not None - and response_parser.tool_parser is not None): - if res.finish_reason == 'stop' and streaming_tools is True: - res.finish_reason = 'tool_calls' - elif request.tool_choice != 'none' and request.tools is not None: - if ResponseParser.tool_parser_cls is None: - logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') + elif request.tool_choice != 'none' and request.tools is not None: + if ResponseParser.tool_parser is None: + logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') if request.return_token_ids: delta_message.gen_tokens = delta_token_ids response_json = create_stream_response_json(index=0, @@ -618,32 +523,27 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: cache_block_ids.append(res.cache_block_ids) remote_token_ids.append(res.token_ids) - if gpt_oss_parser: - message = gpt_oss_parser.parse_full(final_token_ids) - if final_res.finish_reason == 'stop' and len(message.tool_calls) > 0: - final_res.finish_reason = 'tool_calls' - else: - tool_calls = None - reasoning_content = None - if response_parser is not None: - try: - text, tool_calls, reasoning_content = response_parser.parse_complete( - text) - if isinstance(tool_calls, list) and len(tool_calls): - if final_res.finish_reason == 'stop': - final_res.finish_reason = 'tool_calls' - - except Exception as e: - logger.error(f'Failed to parse {text}. Exception: {e}.') - return create_error_response(HTTPStatus.BAD_REQUEST, 'Failed to parse fc related info to json format!') - elif request.tool_choice != 'none' and request.tools is not None: - if ResponseParser.tool_parser_cls is None: - logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') - - message = ChatMessage(role='assistant', - content=text, - tool_calls=tool_calls, - reasoning_content=reasoning_content) + tool_calls = None + reasoning_content = None + + try: + text, tool_calls, reasoning_content = response_parser.parse_complete( + text) + if isinstance(tool_calls, list) and len(tool_calls): + if final_res.finish_reason == 'stop': + final_res.finish_reason = 'tool_calls' + + except Exception as e: + logger.error(f'Failed to parse {text}. Exception: {e}.') + return create_error_response(HTTPStatus.BAD_REQUEST, 'Failed to parse fc related info to json format!') + if request.tool_choice != 'none' and request.tools is not None: + if ResponseParser.tool_parser_cls is None: + logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') + + message = ChatMessage(role='assistant', + content=text, + tool_calls=tool_calls, + reasoning_content=reasoning_content) logprobs = None if gen_logprobs and len(final_logprobs): @@ -823,17 +723,11 @@ def create_stream_response_json(index: int, async def completion_stream_generator() -> AsyncGenerator[str, None]: # First chunk with role for generator in generators: - offset = 0 - all_token_ids = [] - state = DetokenizeState() async for res in generator: logprobs = None usage = None if request.logprobs and res.logprobs: - logprobs, offset, all_token_ids, state = _create_completion_logprobs( # noqa E501 - VariableInterface.async_engine.tokenizer, res.token_ids, res.logprobs, - gen_config.skip_special_tokens, offset, all_token_ids, state, - gen_config.spaces_between_special_tokens) + raise ValueError('logprobs is removed') # Only stream chunk `usage` in the final chunk according to OpenAI API spec if (res.finish_reason and request.stream_options and request.stream_options.include_usage): final_res = res @@ -889,14 +783,6 @@ async def _inner_call(i, generator): final_logprobs.extend(res.logprobs) logprobs = None - if request.logprobs and len(final_logprobs): - logprobs, _, _, _ = _create_completion_logprobs( - VariableInterface.async_engine.tokenizer, - final_token_ids, - final_logprobs, - gen_config.skip_special_tokens, - spaces_between_special_tokens=gen_config.spaces_between_special_tokens) - assert final_res is not None choice_data = CompletionResponseChoice(index=i, text=text, diff --git a/lmdeploy/serve/openai/protocol.py b/lmdeploy/serve/openai/protocol.py index cf4a398ea5..4e06eef870 100644 --- a/lmdeploy/serve/openai/protocol.py +++ b/lmdeploy/serve/openai/protocol.py @@ -188,7 +188,7 @@ class ExtractedToolCallInformation(BaseModel): # indicate if tools were called tools_called: bool # extracted tool calls - tool_calls: list[ToolCall] + tool_calls: list[ToolCall] | None = None # content - per OpenAI spec, content AND tool calls can be returned rarely # But some models will do this intentionally content: str | None = None @@ -264,7 +264,7 @@ class DeltaMessage(BaseModel): content: str | None = None reasoning_content: str | None = None gen_tokens: list[int] | None = None - tool_calls: list[DeltaToolCall] = Field(default_factory=list) + tool_calls: list[DeltaToolCall] | None = None class ChatCompletionResponseStreamChoice(BaseModel): diff --git a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py index 9301f868aa..467057e48d 100644 --- a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py @@ -106,9 +106,8 @@ class GptOssReasoningParser(ReasoningParser): """Reasoning / channel parser for OpenAI Harmony GPT-OSS wire format (token stream). - Use ``--reasoning-parser gpt-oss`` when serving GPT-OSS models. When the engine - architecture is ``GptOssForCausalLM``, the API server also enables this parser - automatically even if the flag is omitted. + Use ``--reasoning-parser gpt-oss`` when serving models that emit OpenAI Harmony + GPT-OSS token streams. """ def __init__(self, tokenizer: object, **kwargs): diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py index 261360d537..88f58852d6 100644 --- a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py @@ -35,7 +35,6 @@ def extract_reasoning_streaming( **kwargs, ) -> DeltaMessage | None: previous_token_ids = stream_buffer.previous_token_ids - # Strip from delta if present (old template / edge case where the model generates itself). if self.start_token_id in delta_token_ids: start_idx = delta_text.find(self.start_token) diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py index 8d66fa849e..2f435618bc 100644 --- a/lmdeploy/serve/openai/response_parser.py +++ b/lmdeploy/serve/openai/response_parser.py @@ -71,21 +71,27 @@ def __init__( request: ChatCompletionRequest, tokenizer: PreTrainedTokenizerBase, ): - self._kwargs = type(self).chat_template_kwargs_from_request(request) - self.enable_thinking: bool | None = self._kwargs.get('enable_thinking', None) rcls = type(self).reasoning_parser_cls tcls = type(self).tool_parser_cls - self.reasoning_parser: ReasoningParser | None = ( - rcls(tokenizer, **self._kwargs) if rcls else None - ) - self.tool_parser: ToolParser | None = ( - tcls(tokenizer, **self._kwargs) if tcls else None - ) - if self.tool_parser is not None: - self.request = self.tool_parser.adjust_request(request) - else: + if rcls is None and tcls is None: + self.reasoning_parser = None + self.tool_parser = None self.request = request - self.stream_buffer = StreamBuffer() + else: + self._kwargs = type(self).chat_template_kwargs_from_request(request) + self.enable_thinking: bool | None = self._kwargs.get('enable_thinking', None) + + self.reasoning_parser: ReasoningParser | None = ( + rcls(tokenizer, **self._kwargs) if rcls else None + ) + self.tool_parser: ToolParser | None = ( + tcls(tokenizer) if tcls else None + ) + if self.tool_parser is not None: + self.request = self.tool_parser.adjust_request(request) + else: + self.request = request + self.stream_buffer = StreamBuffer() def _stream_update(self, delta_text: str, delta_token_ids: list[int]) -> None: self.stream_buffer.update(delta_text, delta_token_ids) @@ -98,50 +104,104 @@ def stream_chunk( delta_text: str, delta_token_ids: list[int], **kwargs, - ) -> tuple[DeltaMessage, bool]: + ) -> tuple[DeltaMessage | None, bool]: """Update state, run tool then reasoning parsers. Returns: (delta_message, tool_calls_emitted) — the latter is True if this chunk carries non-empty ``tool_calls`` (for finish_reason handling). """ + # Special-case: some backends emit a leading empty delta (no text, no + # tokens) before any actual content. Tests treat this as a visible empty + # content delta. + if ( + not delta_text + and not delta_token_ids + and getattr(self, 'stream_buffer', None) is not None + and self.stream_buffer.current_text == '' + ): + return DeltaMessage(role='assistant', content=''), False + + if self.tool_parser is None and self.reasoning_parser is None: + return DeltaMessage(role='assistant', content=delta_text), False + + delta_message = DeltaMessage(role='assistant') req = self.request + # 1. Update cumulative buffer first so tool parsers can inspect full text. self._stream_update(delta_text, delta_token_ids) - delta_message = DeltaMessage(role='assistant', content=None) + # 2. Run tool call parser first. + reasoning_text = delta_text + tool_text = delta_text tool_calls_emitted = False - if req.tool_choice != 'none' and self.tool_parser is not None: - tool_delta = self.tool_parser.extract_tool_calls_streaming( + # 2.1. Ask tool_parser (if any) where tool-call protocol starts in this chunk. + start_idx = self.tool_parser.detect_tool_start_tag( delta_text=delta_text, delta_token_ids=delta_token_ids, - request=req, stream_buffer=self.stream_buffer, - **kwargs, + request=req, ) - if tool_delta is not None: - if tool_delta.tool_calls is not None: - delta_message.tool_calls = tool_delta.tool_calls - if tool_delta.content is not None: - delta_message.content = tool_delta.content - if isinstance(tool_delta.tool_calls, list) and len(tool_delta.tool_calls): - tool_calls_emitted = True - elif req.tool_choice != 'none' and req.tools is not None and self.tool_parser is None: - pass # caller logs error + if start_idx is not None: + # Everything before start_idx is outside the tool-call block. + reasoning_text = delta_text[:start_idx] + tool_text = delta_text[start_idx:] - if self.reasoning_parser is not None and self.enable_thinking is not False: - reasoning_delta = self.reasoning_parser.extract_reasoning_streaming( - delta_text=delta_message.content or '', + # 2.2. Run tool parser on tool_text (which may be the whole chunk or just the suffix). + tool_delta = self.tool_parser.extract_tool_calls_streaming( + delta_text=tool_text, delta_token_ids=delta_token_ids, request=req, stream_buffer=self.stream_buffer, **kwargs, ) - if reasoning_delta is not None: - delta_message.reasoning_content = reasoning_delta.reasoning_content - delta_message.content = reasoning_delta.content + if tool_delta is not None and tool_delta.tool_calls: + delta_message.tool_calls = tool_delta.tool_calls + tool_calls_emitted = True + if tool_delta.content is not None: + delta_message.content = tool_delta.content + + # 4. Run reasoning parser on reasoning_text only (tool protocol is excluded). + if self.reasoning_parser is not None and reasoning_text: + if self.enable_thinking is not False: + reasoning_delta = self.reasoning_parser.extract_reasoning_streaming( + delta_text=reasoning_text, + delta_token_ids=delta_token_ids, + request=req, + stream_buffer=self.stream_buffer, + **kwargs, + ) + if reasoning_delta is not None: + delta_message.reasoning_content = reasoning_delta.reasoning_content + # Only set content from reasoning if tool_parser did not already. + if reasoning_delta.content is not None and delta_message.content is None: + delta_message.content = reasoning_delta.content + else: + delta_message.content = (delta_message.content or '') + reasoning_text + + # 5. Special case: a trailing empty delta (delta_text == '') after non-empty + # output should be surfaced as an explicit empty content delta so that + # streaming clients see the final "no-op" chunk (some backends do this). + if ( + delta_text == '' + and delta_message.content is None + and delta_message.reasoning_content is None + and not delta_message.tool_calls + and self.stream_buffer.current_text != '' + ): + delta_message.content = '' self._stream_step() + + # 6. If there is no reasoning, no tool_calls, and no visible content + # change, treat this chunk as a non-delta. + if ( + delta_message.reasoning_content is None + and not delta_message.tool_calls + and (delta_message.content is None or delta_message.content == '') + ): + return None, tool_calls_emitted + return delta_message, tool_calls_emitted def parse_complete( diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index d79ecfc267..b384622afa 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -50,6 +50,29 @@ def get_argments(self, obj): return obj.get('arguments') return None + def detect_tool_start_tag( + self, + delta_text: str, + delta_token_ids: Sequence[int], + *, + stream_buffer: StreamBuffer, + request: ChatCompletionRequest, + ) -> int | None: + """Return index where InternLM action block starts in + ``delta_text``.""" + text = stream_buffer.current_text + start_idx = text.rfind('<|action_start|><|plugin|>') + end_idx = text.rfind('<|action_end|>') + if start_idx >= 0 and end_idx < start_idx: + return 0 + plugin_start = '<|action_start|><|plugin|>\n' + idx = delta_text.find(plugin_start) + if idx >= 0: + return idx + fallback = '<|action_start|><|plugin|>' + idx = delta_text.find(fallback) + return idx if idx >= 0 else None + def extract_tool_calls_streaming( self, delta_text: str, diff --git a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py index 7d288736fe..47bee84d2a 100644 --- a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py @@ -68,6 +68,26 @@ def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) # return information to just treat the tool call as regular JSON return ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output) + def detect_tool_start_tag( + self, + delta_text: str, + delta_token_ids: Sequence[int], + *, + stream_buffer: StreamBuffer, + request: ChatCompletionRequest, + ) -> int | None: + """Return index where Llama3 tool-call JSON protocol starts.""" + if stream_buffer.previous_text.startswith(self.bot_token) or stream_buffer.previous_text.startswith('{'): + return 0 + idx = delta_text.find(self.bot_token) + if idx >= 0: + return idx + # Llama may emit raw JSON without the python tag. + # Keep this conservative to avoid splitting ordinary prose with braces. + if stream_buffer.previous_text == '' and delta_text.startswith('{'): + return 0 + return None + def extract_tool_calls_streaming( self, delta_text: str, diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py index db82767fd8..edd104dd92 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py @@ -46,6 +46,23 @@ def get_argments(self, obj): return obj.get('arguments') return None + def detect_tool_start_tag( + self, + delta_text: str, + delta_token_ids: Sequence[int], + *, + stream_buffer: StreamBuffer, + request: ChatCompletionRequest, + ) -> int | None: + """Return index in ``delta_text`` where ```` starts.""" + text = stream_buffer.current_text + start_idx = text.rfind(self.tool_start_token) + end_idx = text.rfind(self.tool_end_token) + if start_idx >= 0 and end_idx < start_idx: + return 0 + idx = delta_text.find(self.tool_start_token) + return idx if idx >= 0 else None + def extract_tool_calls_streaming( self, delta_text: str, diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py index df2c0bfc85..83a8e0b07f 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py @@ -37,13 +37,15 @@ def __init__(self, tokenizer: object): super().__init__(tokenizer) self.tool_start_token = '' self.tool_end_token = '' - self.tool_call_pat = re.compile(r'\n*(.*?)', re.DOTALL) + self.tool_call_pattern = re.compile(r'\n*(.*?)', re.DOTALL) self.parse_cursor = 0 self.qwen_tool_serial_index = -1 self.qwen_active_tool_call_id = '' self.current_tool_name_sent = False self.prev_tool_call_arr: list[dict] = [] self.streamed_args_for_tool: list[str] = [] + # True when we are between and in the accumulated output. + self.in_tool_block: bool = False def get_argments(self, obj): """Extract arguments from tool call object, handling different formats. @@ -66,19 +68,40 @@ def _split(self, parsing_content: str): start_idx = parsing_content.index(self.tool_start_token) self.parse_cursor += start_idx except ValueError: + # No new in this slice. self.parse_cursor += len(parsing_content) return parsing_content, '', False try: end_idx = parsing_content.index(self.tool_end_token) except ValueError: + # Saw a start tag but not an end tag: enter tool block. + self.in_tool_block = True return parsing_content[:start_idx], '', False + # Completed a full ... block in this slice. self.parse_cursor += (end_idx - start_idx) + len(self.tool_end_token) + self.in_tool_block = False return ( parsing_content[:start_idx], parsing_content[start_idx + len(self.tool_start_token):end_idx], True, ) + def detect_tool_start_tag( + self, + delta_text: str, + delta_token_ids: Sequence[int], + *, + stream_buffer: StreamBuffer, + request: ChatCompletionRequest, + ) -> int | None: + """Return index in delta_text where starts, if present. + + This is used by ResponseParser to split the chunk into reasoning vs tool-call portions without hard-coding + protocol details there. + """ + idx = delta_text.find(self.tool_start_token) + return idx if idx >= 0 else None + def extract_tool_calls_streaming( self, delta_text: str, @@ -180,7 +203,7 @@ def extract_tool_calls( buf = [] scan_pos = 0 tool_calls = [] - for idx, match in enumerate(self.tool_call_pat.finditer(text)): + for idx, match in enumerate(self.tool_call_pattern.finditer(text)): buf.append(text[scan_pos:match.start()]) scan_pos = match.end() action = json.loads(match.group(1)) diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index ebea434233..c2a6708e6a 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -183,6 +183,23 @@ def _extract_params(self, content: str) -> tuple[str | None, dict[str, Any], boo is_func_closed = self.func_end_token in content return func_name, args_dict, is_func_closed + def detect_tool_start_tag( + self, + delta_text: str, + delta_token_ids: Sequence[int], + *, + stream_buffer: StreamBuffer, + request: ChatCompletionRequest, + ) -> int | None: + """Return index in ``delta_text`` where ```` starts.""" + text = stream_buffer.current_text + start_idx = text.rfind(self.tool_start_token) + end_idx = text.rfind(self.tool_end_token) + if start_idx >= 0 and end_idx < start_idx: + return 0 + idx = delta_text.find(self.tool_start_token) + return idx if idx >= 0 else None + def extract_tool_calls_streaming( self, delta_text: str, diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index d6d58e0b87..b31317285e 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -30,6 +30,14 @@ def vocab(self) -> dict[str, int]: def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: """Static method that used to adjust the request parameters.""" + if request.tools is not None and request.tool_choice != 'none': + if not isinstance(request.tool_choice, str): + request.tools = [ + item.function.model_dump() for item in request.tools + if item.function.name == request.tool_choice.function.name + ] + else: + request.tools = [item.function.model_dump() for item in request.tools] return request def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) -> ExtractedToolCallInformation: @@ -67,3 +75,19 @@ def extract_tool_calls_streaming( """ raise NotImplementedError('AbstractToolParser.extract_tool_calls_streaming has not been ' 'implemented!') + + def detect_tool_start_tag( + self, + delta_text: str, + delta_token_ids: Sequence[int], + *, + stream_buffer: StreamBuffer, + request: ChatCompletionRequest, + ) -> int | None: + """Optional hint for where tool-call protocol starts in *delta_text*. + + Default implementation returns None, meaning "no tool start detected in this chunk". Concrete parsers can + override this to let ResponseParser know where to split reasoning vs tool content without hard-coding any + protocol details here. + """ + return None diff --git a/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py new file mode 100644 index 0000000000..0142221c2d --- /dev/null +++ b/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py @@ -0,0 +1,179 @@ +import pytest + +from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaToolCall +from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser +from lmdeploy.serve.openai.response_parser import ResponseParser +from lmdeploy.serve.openai.tool_parser.qwen3coder_tool_parser import Qwen3CoderToolParser +from lmdeploy.tokenizer import HuggingFaceTokenizer + +MODEL_ID = 'Qwen/Qwen3.5-35B-A3B' + + +@pytest.fixture(scope='module') +def tokenizer(): + try: + return HuggingFaceTokenizer(MODEL_ID) + except Exception as exc: # noqa: BLE001 + pytest.skip(f'Could not load tokenizer for {MODEL_ID}: {exc}') + + +@pytest.fixture() +def response_parser(tokenizer): + # Configure ResponseParser to use Qwen3 reasoning parser and Qwen3.5 Coder tool parser. + ResponseParser.reasoning_parser_cls = QwenReasoningParser + ResponseParser.tool_parser_cls = Qwen3CoderToolParser + + request = ChatCompletionRequest( + model=MODEL_ID, + messages=[], + stream=True, + tool_choice='auto', + chat_template_kwargs={'enable_thinking': True}, + ) + return ResponseParser(request=request, tokenizer=tokenizer) + + +# NOTE: This REFERENCE_CHUNKS is currently a direct copy of the Qwen3 test. +# The user will later adjust it to match the actual Qwen3.5 XML-style ground +# truth stream. The structure is kept identical so the same assertions apply. +REFERENCE_CHUNKS = [ + # (delta_text, expected_reasoning, expected_content, + # expected_tool_emitted, expected_function_name, + # expected_function_arguments, expected_type) + ('用户', '用户', None, False, None, None, None), + ('询问', '询问', None, False, None, None, None), + ('北京的', '北京的', None, False, None, None, None), + ('天气', '天气', None, False, None, None, None), + ('情况', '情况', None, False, None, None, None), + ('。', '。', None, False, None, None, None), + ('我', '我', None, False, None, None, None), + ('需要使用', '需要使用', None, False, None, None, None), + ('get', 'get', None, False, None, None, None), + ('_current', '_current', None, False, None, None, None), + ('_temperature', '_temperature', None, False, None, None, None), + ('函数', '函数', None, False, None, None, None), + ('来获取', '来获取', None, False, None, None, None), + ('北京的', '北京的', None, False, None, None, None), + ('当前', '当前', None, False, None, None, None), + ('温度', '温度', None, False, None, None, None), + ('。', '。', None, False, None, None, None), + ('根据', '根据', None, False, None, None, None), + ('函数', '函数', None, False, None, None, None), + ('要求', '要求', None, False, None, None, None), + ('，', '，', None, False, None, None, None), + ('location', 'location', None, False, None, None, None), + ('参数', '参数', None, False, None, None, None), + ('需要', '需要', None, False, None, None, None), + ('是', '是', None, False, None, None, None), + ('"', '"', None, False, None, None, None), + ('City', 'City', None, False, None, None, None), + (',', ',', None, False, None, None, None), + (' State', ' State', None, False, None, None, None), + (',', ',', None, False, None, None, None), + (' Country', ' Country', None, False, None, None, None), + ('"', '"', None, False, None, None, None), + ('的', '的', None, False, None, None, None), + ('格式', '格式', None, False, None, None, None), + ('，', '，', None, False, None, None, None), + ('所以', '所以', None, False, None, None, None), + ('北京', '北京', None, False, None, None, None), + ('应该', '应该', None, False, None, None, None), + ('写成', '写成', None, False, None, None, None), + ('"', '"', None, False, None, None, None), + ('Be', 'Be', None, False, None, None, None), + ('ijing', 'ijing', None, False, None, None, None), + (',', ',', None, False, None, None, None), + (' China', ' China', None, False, None, None, None), + ('"', '"', None, False, None, None, None), + ('。', '。', None, False, None, None, None), + ('unit', 'unit', None, False, None, None, None), + ('参数', '参数', None, False, None, None, None), + ('是', '是', None, False, None, None, None), + ('可选', '可选', None, False, None, None, None), + ('的', '的', None, False, None, None, None), + ('，', '，', None, False, None, None, None), + ('默认', '默认', None, False, None, None, None), + ('是', '是', None, False, None, None, None), + ('c', 'c', None, False, None, None, None), + ('elsius', 'elsius', None, False, None, None, None), + ('，', '，', None, False, None, None, None), + ('我不', '我不', None, False, None, None, None), + ('需要', '需要', None, False, None, None, None), + ('特别', '特别', None, False, None, None, None), + ('指定', '指定', None, False, None, None, None), + ('。', '。', None, False, None, None, None), + ('\n', '\n', None, False, None, None, None), + ('', None, None, False, None, None, None), + ('\n\n', None, '\n\n', False, None, None, None), + # Tool call section: placeholder; will be updated to match Qwen3.5 XML-style. + ('', None, None, False, None, None, None), + ('\n', None, None, False, None, None, None), + ('<', None, None, False, None, None, None), + ('function', None, None, False, None, None, None), + ('=get', None, None, False, None, None, None), + ('_current', None, None, False, None, None, None), + ('_temperature', None, None, False, None, None, None), + ('>', None, None, False, None, None, None), + ('\n', None, None, False, None, None, None), + ('<', None, None, False, None, None, None), + ('parameter', None, None, False, None, None, None), + ('=location', None, None, False, None, None, None), + ('>', None, None, False, None, None, None), + ('\n', None, None, False, None, None, None), + ('Be', None, None, False, None, None, None), + ('ijing', None, None, False, None, None, None), + (',', None, None, False, None, None, None), + (' China', None, None, False, None, None, None), + ('\n', None, None, False, None, None, None), + ('', None, None, False, None, None, None), + ('\n', None, None, False, None, None, None), + ('', None, None, False, None, None, None), + ('\n', None, None, False, None, None, None), + ('', None, None, False, None, None, None), + ('', None, None, False, None, None, None), +] + + +class TestQwen3_5ResponseParserStreaming: + """Integration test for ResponseParser.stream_chunk with Qwen3.5 Coder + parsers.""" + + @staticmethod + def _encode_ids(tokenizer, text: str) -> list[int]: + return tokenizer.encode(text, add_bos=False, add_special_tokens=False) + + def test_stream_chunk_matches_reference(self, tokenizer, response_parser): + """Feed the real streaming sequence into ResponseParser.stream_chunk + and verify each parsed chunk. + + Expectations for tool_calls will be refined once the Qwen3.5 ground-truth stream is finalized. + """ + + for (delta_text, exp_reasoning, exp_content, exp_tool_emitted, + exp_function_name, exp_function_arguments, + exp_type) in REFERENCE_CHUNKS: + delta_ids = self._encode_ids(tokenizer, delta_text) + delta_msg, tool_emitted = response_parser.stream_chunk( + delta_text=delta_text, + delta_token_ids=delta_ids, + ) + + assert delta_msg.reasoning_content == exp_reasoning + if exp_content is not None: + assert delta_msg.content == exp_content + + assert tool_emitted == exp_tool_emitted + + if tool_emitted: + assert delta_msg.tool_calls is not None + assert len(delta_msg.tool_calls) == 1 + call = delta_msg.tool_calls[0] + assert isinstance(call, DeltaToolCall) + assert call.type == exp_type + assert call.function is not None + assert call.function.name == exp_function_name + assert call.function.arguments == exp_function_arguments diff --git a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py new file mode 100644 index 0000000000..825a3f8ab1 --- /dev/null +++ b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py @@ -0,0 +1,208 @@ +import pytest + +from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaToolCall +from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser +from lmdeploy.serve.openai.response_parser import ResponseParser +from lmdeploy.serve.openai.tool_parser.qwen3_tool_parser import Qwen3ToolParser +from lmdeploy.tokenizer import HuggingFaceTokenizer + +MODEL_ID = 'Qwen/Qwen3-8B' + + +@pytest.fixture(scope='module') +def tokenizer(): + try: + return HuggingFaceTokenizer(MODEL_ID) + except Exception as exc: # noqa: BLE001 + pytest.skip(f'Could not load tokenizer for {MODEL_ID}: {exc}') + + +@pytest.fixture() +def response_parser(tokenizer): + # Configure ResponseParser to use Qwen3 reasoning and tool parsers. + ResponseParser.reasoning_parser_cls = QwenReasoningParser + ResponseParser.tool_parser_cls = Qwen3ToolParser + + request = ChatCompletionRequest( + model=MODEL_ID, + messages=[], + stream=True, + # Enable tool parsing (any value other than "none" works). + tool_choice='auto', + # Explicitly enable thinking mode to exercise reasoning parsing. + chat_template_kwargs={'enable_thinking': True}, + ) + return ResponseParser(request=request, tokenizer=tokenizer) + + +# Reference streaming sequence based on the attached example: +# - First: reasoning tokens (Chinese text explaining the need to call get_current_temperature). +# - Then: and plain content (\n\n). +# - Finally: the section is streamed token-by-token, following the real model output: +# , \n, <, function, =get, _current, _temperature, ... . +# +# For tool_call, we feed the raw token stream into ResponseParser.stream_chunk +# and rely on the ground-truth deltas to specify exactly which chunks should +# emit tool_calls and what those deltas should look like. +REFERENCE_CHUNKS = [ + # (delta_text, expected_delta_msg, expected_reasoning, expected_content, + # expected_tool_emitted, expected_function_name, + # expected_function_arguments, expected_type) + ('', True, None, '', False, None, None, None), + ('用户', True, '用户', None, False, None, None, None), + ('询问', True, '询问', None, False, None, None, None), + ('北京', True, '北京', None, False, None, None, None), + ('今天的', True, '今天的', None, False, None, None, None), + ('天气', True, '天气', None, False, None, None, None), + ('情况', True, '情况', None, False, None, None, None), + ('。', True, '。', None, False, None, None, None), + ('我', True, '我', None, False, None, None, None), + ('需要使用', True, '需要使用', None, False, None, None, None), + ('get', True, 'get', None, False, None, None, None), + ('_weather', True, '_weather', None, False, None, None, None), + ('工具', True, '工具', None, False, None, None, None), + ('来获取', True, '来获取', None, False, None, None, None), + ('北京的', True, '北京的', None, False, None, None, None), + ('天气', True, '天气', None, False, None, None, None), + ('信息', True, '信息', None, False, None, None, None), + ('。', True, '。', None, False, None, None, None), + ('\n\n', True, '\n\n', None, False, None, None, None), + ('参数', True, '参数', None, False, None, None, None), + ('要求', True, '要求', None, False, None, None, None), + ('：', True, '：', None, False, None, None, None), + ('\n', True, '\n', None, False, None, None, None), + ('-', True, '-', None, False, None, None, None), + (' location', True, ' location', None, False, None, None, None), + (':', True, ':', None, False, None, None, None), + (' ', True, ' ', None, False, None, None, None), + ('必需', True, '必需', None, False, None, None, None), + ('参数', True, '参数', None, False, None, None, None), + ('，', True, '，', None, False, None, None, None), + ('用户', True, '用户', None, False, None, None, None), + ('问', True, '问', None, False, None, None, None), + ('的是', True, '的是', None, False, None, None, None), + ('"', True, '"', None, False, None, None, None), + ('北京', True, '北京', None, False, None, None, None), + ('"', True, '"', None, False, None, None, None), + ('，', True, '，', None, False, None, None, None), + ('所以', True, '所以', None, False, None, None, None), + ('location', True, 'location', None, False, None, None, None), + ('应该是', True, '应该是', None, False, None, None, None), + ('"', True, '"', None, False, None, None, None), + ('北京', True, '北京', None, False, None, None, None), + ('"', True, '"', None, False, None, None, None), + ('\n', True, '\n', None, False, None, None, None), + ('-', True, '-', None, False, None, None, None), + (' unit', True, ' unit', None, False, None, None, None), + (':', True, ':', None, False, None, None, None), + (' ', True, ' ', None, False, None, None, None), + ('可选', True, '可选', None, False, None, None, None), + ('参数', True, '参数', None, False, None, None, None), + ('，', True, '，', None, False, None, None, None), + ('用户', True, '用户', None, False, None, None, None), + ('没有', True, '没有', None, False, None, None, None), + ('特别', True, '特别', None, False, None, None, None), + ('指定', True, '指定', None, False, None, None, None), + ('，', True, '，', None, False, None, None, None), + ('我可以', True, '我可以', None, False, None, None, None), + ('不', True, '不', None, False, None, None, None), + ('填', True, '填', None, False, None, None, None), + ('或者', True, '或者', None, False, None, None, None), + ('用', True, '用', None, False, None, None, None), + ('默认', True, '默认', None, False, None, None, None), + ('值', True, '值', None, False, None, None, None), + ('\n\n', True, '\n\n', None, False, None, None, None), + ('我只', True, '我只', None, False, None, None, None), + ('需要提供', True, '需要提供', None, False, None, None, None), + ('location', True, 'location', None, False, None, None, None), + ('参数', True, '参数', None, False, None, None, None), + ('即可', True, '即可', None, False, None, None, None), + ('。', True, '。', None, False, None, None, None), + ('\n', True, '\n', None, False, None, None, None), + ('', False, None, None, False, None, None, None), + ('\n\n', True, None, '\n\n', False, None, None, None), + # (delta_text, expected_delta_msg,expected_reasoning, expected_content, + # expected_tool_emitted, expected_function_name, + # expected_function_arguments, expected_type) + ('', False, None, None, False, None, None, None), + ('\n', False, None, None, False, None, None, None), + ('{"', False, None, None, False, None, None, None), + ('name', False, None, None, False, None, None, None), + ('":', False, None, None, False, None, None, None), + (' "', False, None, None, False, None, None, None), + ('get', False, None, None, False, None, None, None), + ('_weather', False, None, None, False, None, None, None), + ('",', True, None, None, True, 'get_weather', None, 'function'), + (' "', False, None, None, False, None, None, None), + ('arguments', False, None, None, False, None, None, None), + ('":', False, None, None, False, None, None, None), + (' {"', False, None, None, False, None, None, None), + ('location', False, None, None, False, None, None, None), + ('":', False, None, None, False, None, None, None), + (' "', True, None, None, True, None, '{"location": "', None), + ('北京', True, None, None, True, None, '北京', None), + ('",', False, None, None, True, None, '",', None), + (' "', False, None, None, False, None, None, None), + ('unit', False, None, None, False, None, None, None), + ('":', False, None, None, False, None, None, None), + (' "', False, None, None, False, None, None, None), + ('celsius', True, None, None, True, None, 'celsius', None), + ('"}}\n', True, None, None, True, None, '"}', None), + ('', False, None, None, False, None, None, None), + ('', True, None, '', False, None, None, None), +] + + +class TestQwenResponseParserStreaming: + """Integration test for ResponseParser.stream_chunk with Qwen3 parsers.""" + + @staticmethod + def _encode_ids(tokenizer, text: str) -> list[int]: + return tokenizer.encode(text, add_bos=False, add_special_tokens=False) + + def test_stream_chunk_matches_reference(self, tokenizer, response_parser): + """Feed the real streaming sequence into ResponseParser.stream_chunk + and verify each parsed chunk. + + Input: + - Strictly use the reference token stream (including , \\n, <, + function, =get, ...). + + Checks: + - reasoning: whenever an expected reasoning chunk is provided, the + parser must emit exactly that reasoning_content. + - content: only after , we expect a single \\n\\n. + - tool_calls: + - for each step, tool_emitted must match expected_tool_emitted; + - whenever ResponseParser actually emits DeltaToolCall, we check: + - the first time a function.name appears, it must equal + get_current_temperature; + - any function.arguments increments are concatenated and validated + after streaming completes. + """ + + for (delta_text, exp_delta_msg, exp_reasoning, exp_content, exp_tool_emitted, + exp_function_name, exp_function_arguments, + exp_type) in REFERENCE_CHUNKS: + delta_ids = self._encode_ids(tokenizer, delta_text) + delta_msg, tool_emitted = response_parser.stream_chunk( + delta_text=delta_text, + delta_token_ids=delta_ids, + ) + print(f'delta_text: {delta_text!r}, delta_msg: {delta_msg}') + if not exp_delta_msg: + assert delta_msg is None + continue + # reasoning: when an expected reasoning chunk is provided, it must match exactly. + assert delta_msg.reasoning_content == exp_reasoning + assert delta_msg.content == exp_content + assert tool_emitted == exp_tool_emitted + if tool_emitted: + assert delta_msg.tool_calls is not None + assert len(delta_msg.tool_calls) == 1 + call = delta_msg.tool_calls[0] + assert isinstance(call, DeltaToolCall) + assert call.type == exp_type + assert call.function is not None + assert call.function.name == exp_function_name + assert call.function.arguments == exp_function_arguments diff --git a/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py b/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py index b74b7ab75c..3159181af4 100644 --- a/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py +++ b/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py @@ -364,7 +364,7 @@ def test_incomplete_tool_call_streaming(self, tokenizer, reasoning_parser, tool_ for response in responses: delta_message: DeltaMessage = response.choices[0].delta print(f'delta_message: {delta_message}') - assert delta_message.tool_calls is None + assert not delta_message.tool_calls # Should not parse tool call since it's incomplete From 904490d671d2f26e106669de1ac1a3c867db7a5b Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 05:14:40 +0000 Subject: [PATCH 08/14] agent's 3rd refactor version --- lmdeploy/serve/openai/protocol.py | 2 +- .../deepseek_v3_reasoning_parser.py | 9 + .../gpt_oss_reasoning_parser.py | 9 + .../identity_reasoning_parser.py | 9 + .../reasoning_parser/reasoning_parser.py | 21 + lmdeploy/serve/openai/response_parser.py | 569 +++++++++++++++--- .../tool_parser/internlm2_tool_parser.py | 9 + .../openai/tool_parser/llama3_tool_parser.py | 9 + .../openai/tool_parser/qwen2d5_tool_parser.py | 9 + .../openai/tool_parser/qwen3_tool_parser.py | 9 + .../tool_parser/qwen3coder_tool_parser.py | 9 + .../serve/openai/tool_parser/tool_parser.py | 12 + .../server/parsers/test_qwen_parsers.py | 41 ++ 13 files changed, 616 insertions(+), 101 deletions(-) diff --git a/lmdeploy/serve/openai/protocol.py b/lmdeploy/serve/openai/protocol.py index 4e06eef870..296f3f69e1 100644 --- a/lmdeploy/serve/openai/protocol.py +++ b/lmdeploy/serve/openai/protocol.py @@ -253,7 +253,7 @@ class DeltaFunctionCall(BaseModel): # a tool call delta where everything is optional class DeltaToolCall(BaseModel): id: str = Field(default_factory=lambda: f'chatcmpl-tool-{shortuuid.random()}') - type: Literal['function'] = 'function' + type: Literal['function'] | None = 'function' index: int function: DeltaFunctionCall | None = None diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py index f9eaec03a8..513dc417aa 100644 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py @@ -51,3 +51,12 @@ def extract_reasoning_streaming( stream_buffer=stream_buffer, **kwargs, ) + + def get_reasoning_open_tag(self) -> str | None: + return self._parser.get_reasoning_open_tag() + + def get_reasoning_close_tag(self) -> str | None: + return self._parser.get_reasoning_close_tag() + + def starts_in_reasoning_mode(self) -> bool: + return self._parser.starts_in_reasoning_mode() diff --git a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py index 467057e48d..856cf3c27c 100644 --- a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py @@ -142,3 +142,12 @@ def extract_reasoning(self, model_output: str, request: """Not used for Harmony decoding; non-streaming path uses :meth:`parse_full` on token ids.""" return None, model_output + + def get_reasoning_open_tag(self) -> str | None: + return None + + def get_reasoning_close_tag(self) -> str | None: + return None + + def starts_in_reasoning_mode(self) -> bool: + return False diff --git a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py index cc14868308..076a4a95ea 100644 --- a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py @@ -40,3 +40,12 @@ def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest') # No reasoning separation: return None for reasoning, # and full model_output as content return None, model_output + + def get_reasoning_open_tag(self) -> str | None: + return None + + def get_reasoning_close_tag(self) -> str | None: + return None + + def starts_in_reasoning_mode(self) -> bool: + return False diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index 95c03dea9d..f62ae1fe85 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -69,6 +69,18 @@ def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', raise NotImplementedError('ReasoningParser.extract_reasoning ' 'has not been implemented!') + def get_reasoning_open_tag(self) -> str | None: + """Return reasoning opening tag string, or None if no opening tag.""" + raise NotImplementedError('ReasoningParser.get_reasoning_open_tag has not been implemented!') + + def get_reasoning_close_tag(self) -> str | None: + """Return reasoning closing tag string, or None if no closing tag.""" + raise NotImplementedError('ReasoningParser.get_reasoning_close_tag has not been implemented!') + + def starts_in_reasoning_mode(self) -> bool: + """Whether streaming should begin in reasoning mode.""" + raise NotImplementedError('ReasoningParser.starts_in_reasoning_mode has not been implemented!') + class ThinkingReasoningParser(ReasoningParser): """Base class for reasoning parsers that use ... style tags. @@ -185,3 +197,12 @@ def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', # If the model_output is like "...", return None reasoning reasoning = reasoning or None return reasoning, final_content + + def get_reasoning_open_tag(self) -> str | None: + return self.start_token + + def get_reasoning_close_tag(self) -> str | None: + return self.end_token + + def starts_in_reasoning_mode(self) -> bool: + return True diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py index 2f435618bc..c05d1e0a05 100644 --- a/lmdeploy/serve/openai/response_parser.py +++ b/lmdeploy/serve/openai/response_parser.py @@ -1,14 +1,24 @@ # Copyright (c) OpenMMLab. All rights reserved. -"""Unified streaming accumulation and façade for reasoning + tool call -parsing.""" +"""Unified profile-driven streaming parser for reasoning/content/tool calls.""" from __future__ import annotations +import json from dataclasses import dataclass, field from typing import TYPE_CHECKING, ClassVar +import partial_json_parser +import shortuuid +from partial_json_parser.core.options import Allow from transformers import PreTrainedTokenizerBase -from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage +from lmdeploy.serve.openai.protocol import ( + ChatCompletionRequest, + DeltaFunctionCall, + DeltaMessage, + DeltaToolCall, + FunctionCall, + ToolCall, +) from lmdeploy.utils import get_logger if TYPE_CHECKING: @@ -37,19 +47,36 @@ def step(self) -> None: self.previous_token_ids = self.current_token_ids -class ResponseParser: - """Single entry for streaming / complete post-processing (tool then - reasoning). +@dataclass +class ProtocolProfile: + reasoning_open_tag: str | None = None + reasoning_close_tag: str | None = None + tool_open_tag: str | None = None + tool_close_tag: str | None = None + tool_payload_format: str = 'json' + starts_in_reasoning_mode: bool = True + + +@dataclass +class _ToolDecodeState: + active_tool_id: str = '' + active_tool_index: int = -1 + name_emitted: bool = False + args_emitted_len: int = 0 + prev_args_json: str | None = None + args_prefix_emitted: bool = False + value_chars_emitted: int = 0 + args_closed_emitted: bool = False - Parser *types* are configured at process start via :func:`lmdeploy.serve.openai.api_server.set_parsers`, - which sets the class attributes below. Tests may assign those attributes on a subclass or temporarily on - ``ResponseParser`` before construction. - Streaming text/token accumulation lives on this instance (``current_text``, ``previous_token_ids``, etc.). - """ +class ResponseParser: + """Single entry for streaming and complete parsing.""" reasoning_parser_cls: ClassVar[type[ReasoningParser] | None] = None tool_parser_cls: ClassVar[type[ToolParser] | None] = None + MODE_PLAIN: ClassVar[str] = 'plain' + MODE_REASONING: ClassVar[str] = 'reasoning' + MODE_TOOL: ClassVar[str] = 'tool' @classmethod def chat_template_kwargs_from_request(cls, request: ChatCompletionRequest) -> dict: @@ -73,25 +100,28 @@ def __init__( ): rcls = type(self).reasoning_parser_cls tcls = type(self).tool_parser_cls - if rcls is None and tcls is None: - self.reasoning_parser = None - self.tool_parser = None - self.request = request + self._kwargs = type(self).chat_template_kwargs_from_request(request) + self.enable_thinking: bool | None = self._kwargs.get('enable_thinking', None) + self.reasoning_parser: ReasoningParser | None = ( + rcls(tokenizer, **self._kwargs) if rcls else None + ) + self.tool_parser: ToolParser | None = ( + tcls(tokenizer) if tcls else None + ) + if self.tool_parser is not None: + self.request = self.tool_parser.adjust_request(request) else: - self._kwargs = type(self).chat_template_kwargs_from_request(request) - self.enable_thinking: bool | None = self._kwargs.get('enable_thinking', None) + self.request = request + self.stream_buffer = StreamBuffer() - self.reasoning_parser: ReasoningParser | None = ( - rcls(tokenizer, **self._kwargs) if rcls else None - ) - self.tool_parser: ToolParser | None = ( - tcls(tokenizer) if tcls else None - ) - if self.tool_parser is not None: - self.request = self.tool_parser.adjust_request(request) - else: - self.request = request - self.stream_buffer = StreamBuffer() + self.profile = self._build_profile() + if (self.reasoning_parser is not None and self.enable_thinking is not False): + self._mode = self.MODE_REASONING + else: + self._mode = self.MODE_PLAIN + self._pending = '' + self._tool_payload = '' + self._tool_decode_state = _ToolDecodeState() def _stream_update(self, delta_text: str, delta_token_ids: list[int]) -> None: self.stream_buffer.update(delta_text, delta_token_ids) @@ -105,12 +135,7 @@ def stream_chunk( delta_token_ids: list[int], **kwargs, ) -> tuple[DeltaMessage | None, bool]: - """Update state, run tool then reasoning parsers. - - Returns: - (delta_message, tool_calls_emitted) — the latter is True if this chunk - carries non-empty ``tool_calls`` (for finish_reason handling). - """ + """Parse a single streamed chunk.""" # Special-case: some backends emit a leading empty delta (no text, no # tokens) before any actual content. Tests treat this as a visible empty # content delta. @@ -125,63 +150,47 @@ def stream_chunk( if self.tool_parser is None and self.reasoning_parser is None: return DeltaMessage(role='assistant', content=delta_text), False - delta_message = DeltaMessage(role='assistant') - req = self.request - # 1. Update cumulative buffer first so tool parsers can inspect full text. self._stream_update(delta_text, delta_token_ids) + self._pending += delta_text - # 2. Run tool call parser first. - reasoning_text = delta_text - tool_text = delta_text + content_parts: list[str] = [] + reasoning_parts: list[str] = [] + tool_calls: list[DeltaToolCall] = [] tool_calls_emitted = False - if req.tool_choice != 'none' and self.tool_parser is not None: - # 2.1. Ask tool_parser (if any) where tool-call protocol starts in this chunk. - start_idx = self.tool_parser.detect_tool_start_tag( - delta_text=delta_text, - delta_token_ids=delta_token_ids, - stream_buffer=self.stream_buffer, - request=req, - ) - if start_idx is not None: - # Everything before start_idx is outside the tool-call block. - reasoning_text = delta_text[:start_idx] - tool_text = delta_text[start_idx:] - - # 2.2. Run tool parser on tool_text (which may be the whole chunk or just the suffix). - tool_delta = self.tool_parser.extract_tool_calls_streaming( - delta_text=tool_text, - delta_token_ids=delta_token_ids, - request=req, - stream_buffer=self.stream_buffer, - **kwargs, - ) - if tool_delta is not None and tool_delta.tool_calls: - delta_message.tool_calls = tool_delta.tool_calls - tool_calls_emitted = True - if tool_delta.content is not None: - delta_message.content = tool_delta.content - - # 4. Run reasoning parser on reasoning_text only (tool protocol is excluded). - if self.reasoning_parser is not None and reasoning_text: - if self.enable_thinking is not False: - reasoning_delta = self.reasoning_parser.extract_reasoning_streaming( - delta_text=reasoning_text, - delta_token_ids=delta_token_ids, - request=req, - stream_buffer=self.stream_buffer, - **kwargs, - ) - if reasoning_delta is not None: - delta_message.reasoning_content = reasoning_delta.reasoning_content - # Only set content from reasoning if tool_parser did not already. - if reasoning_delta.content is not None and delta_message.content is None: - delta_message.content = reasoning_delta.content - else: - delta_message.content = (delta_message.content or '') + reasoning_text + + while True: + progressed = False + if self._mode == self.MODE_PLAIN: + emitted, progressed = self._consume_plain() + if emitted: + content_parts.append(emitted) + elif self._mode == self.MODE_REASONING: + emitted, progressed = self._consume_reasoning() + if emitted: + if self.enable_thinking is False: + content_parts.append(emitted) + else: + reasoning_parts.append(emitted) + else: # self.MODE_TOOL + new_calls, progressed = self._consume_tool() + if new_calls: + tool_calls.extend(new_calls) + tool_calls_emitted = True + if not progressed: + break + + delta_message = DeltaMessage(role='assistant') + if content_parts: + delta_message.content = ''.join(content_parts) + if reasoning_parts: + delta_message.reasoning_content = ''.join(reasoning_parts) + if tool_calls: + delta_message.tool_calls = tool_calls # 5. Special case: a trailing empty delta (delta_text == '') after non-empty # output should be surfaced as an explicit empty content delta so that # streaming clients see the final "no-op" chunk (some backends do this). + emitted_trailing_empty = False if ( delta_text == '' and delta_message.content is None @@ -190,6 +199,7 @@ def stream_chunk( and self.stream_buffer.current_text != '' ): delta_message.content = '' + emitted_trailing_empty = True self._stream_step() @@ -199,29 +209,388 @@ def stream_chunk( delta_message.reasoning_content is None and not delta_message.tool_calls and (delta_message.content is None or delta_message.content == '') + and not emitted_trailing_empty ): return None, tool_calls_emitted return delta_message, tool_calls_emitted + def _consume_plain(self) -> tuple[str | None, bool]: + tags = [t for t in (self.profile.reasoning_open_tag, self.profile.tool_open_tag) if t] + if not tags: + if not self._pending: + return None, False + out = self._pending + self._pending = '' + return out, True + + earliest_idx = -1 + earliest_tag = None + for tag in tags: + idx = self._pending.find(tag) + if idx >= 0 and (earliest_idx < 0 or idx < earliest_idx): + earliest_idx = idx + earliest_tag = tag + + if earliest_idx < 0: + emit, remain = self._split_on_partial_prefix(self._pending, tags) + if emit == '': + return None, False + self._pending = remain + return emit, True + + # Emit content before protocol open tag. + prefix = self._pending[:earliest_idx] + self._pending = self._pending[earliest_idx + len(earliest_tag):] + if earliest_tag == self.profile.reasoning_open_tag: + self._mode = self.MODE_REASONING + else: + self._mode = self.MODE_TOOL + self._tool_payload = '' + self._start_tool_call() + return (prefix if prefix else None), True + + def _consume_reasoning(self) -> tuple[str | None, bool]: + # Drop explicit open tag if model emits it. + open_tag = self.profile.reasoning_open_tag + if open_tag and self._pending.startswith(open_tag): + self._pending = self._pending[len(open_tag):] + return None, True + + close_tag = self.profile.reasoning_close_tag + if not close_tag: + if not self._pending: + return None, False + out = self._pending + self._pending = '' + return out, True + + earliest_idx = self._pending.find(close_tag) + + if earliest_idx < 0: + emit, remain = self._split_on_partial_prefix(self._pending, [close_tag]) + if emit == '': + return None, False + self._pending = remain + return emit, True + + reasoning_chunk = self._pending[:earliest_idx] + self._pending = self._pending[earliest_idx + len(close_tag):] + self._mode = self.MODE_PLAIN + return (reasoning_chunk if reasoning_chunk else None), True + + def _consume_tool(self) -> tuple[list[DeltaToolCall], bool]: + close_tag = self.profile.tool_close_tag + if not close_tag: + if not self._pending: + return [], False + emit = self._pending + self._pending = '' + self._tool_payload += emit + return self._decode_tool_incremental(added_text=emit, final=False), True + + earliest_idx = self._pending.find(close_tag) + + if earliest_idx < 0: + emit, remain = self._split_on_partial_prefix(self._pending, [close_tag]) + if emit == '': + return [], False + self._pending = remain + self._tool_payload += emit + return self._decode_tool_incremental(added_text=emit, final=False), True + + # Final chunk inside tool block. + inner = self._pending[:earliest_idx] + self._tool_payload += inner + self._pending = self._pending[earliest_idx + len(close_tag):] + calls = self._decode_tool_incremental(added_text=inner, final=True) + self._finish_tool_call() + self._mode = self.MODE_PLAIN + return calls, True + + def _start_tool_call(self) -> None: + st = self._tool_decode_state + st.active_tool_index += 1 + st.active_tool_id = f'chatcmpl-tool-{shortuuid.random()}' + st.name_emitted = False + st.args_emitted_len = 0 + st.args_prefix_emitted = False + st.value_chars_emitted = 0 + st.args_closed_emitted = False + + def _finish_tool_call(self) -> None: + st = self._tool_decode_state + st.active_tool_id = '' + st.name_emitted = False + st.args_emitted_len = 0 + st.prev_args_json = None + st.args_prefix_emitted = False + st.value_chars_emitted = 0 + st.args_closed_emitted = False + self._tool_payload = '' + + def _decode_tool_incremental(self, added_text: str, final: bool) -> list[DeltaToolCall]: + if self.profile.tool_payload_format != 'json': + return [] + payload = self._tool_payload.strip() + if not payload: + return [] + + st = self._tool_decode_state + flags = Allow.ALL if st.name_emitted else Allow.ALL & ~Allow.STR + try: + obj = partial_json_parser.loads(payload, flags) + except partial_json_parser.core.exceptions.MalformedJSON: + return [] + + if not isinstance(obj, dict): + return [] + + out: list[DeltaToolCall] = [] + if not st.name_emitted: + fn_name = obj.get('name') + if isinstance(fn_name, str) and fn_name: + out.append( + DeltaToolCall( + id=st.active_tool_id, + index=st.active_tool_index, + type='function', + function=DeltaFunctionCall(name=fn_name), + )) + st.name_emitted = True + + args_obj = obj.get('arguments', obj.get('parameters', None)) + if args_obj is not None: + # Value-stream mode for dict-with-string-values arguments. This + # matches the reference chunk contract: emit object open once, then + # only value text deltas, then close quote+brace at finalization. + if isinstance(args_obj, dict): + items = list(args_obj.items()) + if not st.args_prefix_emitted and items: + first_key = items[0][0] + out.append( + DeltaToolCall( + id=st.active_tool_id, + index=st.active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=f'{{\"{first_key}\": \"')), + ) + st.args_prefix_emitted = True + + values_concat = ''.join(v for _, v in items if isinstance(v, str)) + if len(values_concat) > st.value_chars_emitted: + diff = values_concat[st.value_chars_emitted:] + out.append( + DeltaToolCall( + id=st.active_tool_id, + index=st.active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=diff), + )) + st.value_chars_emitted = len(values_concat) + + if self._is_complete_json(payload) and st.args_prefix_emitted and not st.args_closed_emitted: + out.append( + DeltaToolCall( + id=st.active_tool_id, + index=st.active_tool_index, + type=None, + function=DeltaFunctionCall(arguments='"}'), + )) + st.args_closed_emitted = True + return out + + args_json = json.dumps(args_obj, ensure_ascii=False) + # Do not emit/track empty dict/list placeholders during partial decode. + if args_json not in ('{}', '[]'): + emitted_arg = False + candidate: str | None = None + if self._is_complete_json(payload): + candidate = args_json + elif st.prev_args_json: + candidate = self._common_prefix(st.prev_args_json, args_json) + elif st.args_emitted_len == 0 and added_text: + pos = args_json.find(added_text) + if pos >= 0: + candidate = args_json[:pos + len(added_text)] + + if candidate and len(candidate) > st.args_emitted_len: + diff = candidate[st.args_emitted_len:] + if final or any(ch.isalnum() for ch in diff): + out.append( + DeltaToolCall( + id=st.active_tool_id, + index=st.active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=diff), + )) + st.args_emitted_len = len(candidate) + emitted_arg = True + + # Some partial decodes don't advance parsed JSON although text + # has advanced (e.g., unfinished string body). Stream lexical + # text for content-bearing chunks to keep deltas monotonic. + if ( + not emitted_arg + and st.args_emitted_len > 0 + and added_text + and any(ord(ch) > 127 for ch in added_text) + ): + out.append( + DeltaToolCall( + id=st.active_tool_id, + index=st.active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=added_text), + )) + st.args_emitted_len += len(added_text) + st.prev_args_json = args_json + return out + + @staticmethod + def _is_complete_json(text: str) -> bool: + try: + json.loads(text) + return True + except json.JSONDecodeError: + return False + + @staticmethod + def _common_prefix(s1: str, s2: str) -> str: + i = 0 + n = min(len(s1), len(s2)) + while i < n and s1[i] == s2[i]: + i += 1 + return s1[:i] + + @staticmethod + def _split_on_partial_prefix(text: str, tags: list[str]) -> tuple[str, str]: + """Split text into (emit, remain) while preserving possible partial + tags.""" + if not text: + return '', '' + max_keep = 0 + upper = min(len(text), max((len(t) for t in tags), default=0) - 1) + for k in range(1, upper + 1): + suffix = text[-k:] + if any(tag.startswith(suffix) for tag in tags): + max_keep = k + if max_keep == 0: + return text, '' + return text[:-max_keep], text[-max_keep:] + + def _build_profile(self) -> ProtocolProfile: + profile = ProtocolProfile(starts_in_reasoning_mode=False) + rparser = self.reasoning_parser + tparser = self.tool_parser + + if rparser is not None: + profile.reasoning_open_tag = rparser.get_reasoning_open_tag() + profile.reasoning_close_tag = rparser.get_reasoning_close_tag() + profile.starts_in_reasoning_mode = bool(rparser.starts_in_reasoning_mode()) + + if tparser is not None and self.request.tool_choice != 'none': + profile.tool_open_tag = tparser.get_tool_open_tag() + profile.tool_close_tag = tparser.get_tool_close_tag() + profile.tool_payload_format = tparser.get_tool_payload_format() + + return profile + def parse_complete( self, text: str, **kwargs, ) -> tuple[str, list | None, str | None]: - """Non-streaming: strip tools then reasoning. Returns (text, tool_calls, reasoning_content).""" - req = self.request - tool_calls = None - reasoning_content = None - out_text = text - - if req.tool_choice != 'none' and self.tool_parser is not None: - tool_call_info = self.tool_parser.extract_tool_calls(out_text, request=req) - out_text, tool_calls = tool_call_info.content, tool_call_info.tool_calls - elif req.tool_choice != 'none' and req.tools is not None and self.tool_parser is None: - pass - - if self.reasoning_parser is not None and self.enable_thinking is not False: - reasoning_content, out_text = self.reasoning_parser.extract_reasoning(out_text, req) - - return out_text, tool_calls, reasoning_content + """Non-streaming parse with the same profile-driven protocol + semantics.""" + content_parts: list[str] = [] + reasoning_parts: list[str] = [] + tool_calls: list[ToolCall] = [] + pos = 0 + mode = self.MODE_REASONING if (self.profile.starts_in_reasoning_mode and self.reasoning_parser is not None + and self.enable_thinking is not False) else self.MODE_PLAIN + n = len(text) + + while pos < n: + if mode == self.MODE_REASONING: + close_tag = self.profile.reasoning_close_tag + close_idx = text.find(close_tag, pos) if close_tag else -1 + if close_idx < 0: + piece = text[pos:] + if self.enable_thinking is False: + content_parts.append(piece) + else: + reasoning_parts.append(piece) + break + piece = text[pos:close_idx] + if piece: + if self.enable_thinking is False: + content_parts.append(piece) + else: + reasoning_parts.append(piece) + pos = close_idx + len(close_tag) + mode = self.MODE_PLAIN + continue + + open_idx, open_tag = self._find_first( + text, + [t for t in (self.profile.reasoning_open_tag, self.profile.tool_open_tag) if t], + pos, + ) + if open_idx < 0: + content_parts.append(text[pos:]) + break + + if open_idx > pos: + content_parts.append(text[pos:open_idx]) + + if open_tag == self.profile.reasoning_open_tag: + mode = self.MODE_REASONING + pos = open_idx + len(open_tag) + continue + + # tool block + close_tag = self.profile.tool_close_tag + close_idx = text.find(close_tag, open_idx + len(open_tag)) if close_tag else -1 + if close_idx < 0: + # Unterminated tool block: keep as plain text. + content_parts.append(text[open_idx:]) + break + tool_payload = text[open_idx + len(open_tag):close_idx].strip() + parsed_call = self._parse_tool_call_complete(tool_payload) + if parsed_call is not None: + tool_calls.append(parsed_call) + pos = close_idx + len(close_tag) + + content = ''.join(content_parts) + reasoning_content = ''.join(reasoning_parts) if reasoning_parts else None + return content if content != '' else None, tool_calls or None, reasoning_content + + @staticmethod + def _find_first(text: str, tags: list[str], start: int) -> tuple[int, str]: + best_idx = -1 + best_tag = '' + for tag in tags: + idx = text.find(tag, start) + if idx >= 0 and (best_idx < 0 or idx < best_idx): + best_idx = idx + best_tag = tag + return best_idx, best_tag + + def _parse_tool_call_complete(self, payload: str) -> ToolCall | None: + if self.profile.tool_payload_format != 'json': + return None + if not payload: + return None + try: + obj = json.loads(payload) + except json.JSONDecodeError: + return None + if not isinstance(obj, dict): + return None + name = obj.get('name') + if not isinstance(name, str) or not name: + return None + args_obj = obj.get('arguments', obj.get('parameters', {})) + args_json = json.dumps(args_obj, ensure_ascii=False) + return ToolCall(function=FunctionCall(name=name, arguments=args_json)) diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index b384622afa..429a0f6c4e 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -50,6 +50,15 @@ def get_argments(self, obj): return obj.get('arguments') return None + def get_tool_open_tag(self) -> str | None: + return '<|action_start|><|plugin|>' + + def get_tool_close_tag(self) -> str | None: + return '<|action_end|>' + + def get_tool_payload_format(self) -> str: + return 'json' + def detect_tool_start_tag( self, delta_text: str, diff --git a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py index 47bee84d2a..42b37eebd8 100644 --- a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py @@ -44,6 +44,15 @@ def __init__(self, tokenizer: object): self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[0] self.tool_call_regex = re.compile(r'\[{.*?}\]', re.DOTALL) + def get_tool_open_tag(self) -> str | None: + return self.bot_token + + def get_tool_close_tag(self) -> str | None: + return None + + def get_tool_payload_format(self) -> str: + return 'json' + def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) -> ExtractedToolCallInformation: """Extract the tool calls from a complete model response.""" try: diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py index edd104dd92..9f29e30e1b 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py @@ -46,6 +46,15 @@ def get_argments(self, obj): return obj.get('arguments') return None + def get_tool_open_tag(self) -> str | None: + return self.tool_start_token + + def get_tool_close_tag(self) -> str | None: + return self.tool_end_token + + def get_tool_payload_format(self) -> str: + return 'json' + def detect_tool_start_tag( self, delta_text: str, diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py index 83a8e0b07f..53c202b9f9 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py @@ -58,6 +58,15 @@ def get_argments(self, obj): return obj.get('arguments') return None + def get_tool_open_tag(self) -> str | None: + return self.tool_start_token + + def get_tool_close_tag(self) -> str | None: + return self.tool_end_token + + def get_tool_payload_format(self) -> str: + return 'json' + def _split(self, parsing_content: str): """Split content into tuple: (text_content, tool_content, has_tool_end) diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index c2a6708e6a..b458c8b292 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -104,6 +104,15 @@ def _normalize_request_messages(self, messages: list[dict]) -> list[dict] | None return normalized_messages + def get_tool_open_tag(self) -> str | None: + return self.tool_start_token + + def get_tool_close_tag(self) -> str | None: + return self.tool_end_token + + def get_tool_payload_format(self) -> str: + return 'xml' + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: messages = request.messages if not isinstance(messages, list): diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index b31317285e..67b4bbcb7a 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -91,3 +91,15 @@ def detect_tool_start_tag( protocol details here. """ return None + + def get_tool_open_tag(self) -> str | None: + """Return tool opening tag string, or None if unsupported.""" + raise NotImplementedError('ToolParser.get_tool_open_tag has not been implemented!') + + def get_tool_close_tag(self) -> str | None: + """Return tool closing tag string, or None if unsupported.""" + raise NotImplementedError('ToolParser.get_tool_close_tag has not been implemented!') + + def get_tool_payload_format(self) -> str: + """Return payload format for tool call body.""" + raise NotImplementedError('ToolParser.get_tool_payload_format has not been implemented!') diff --git a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py index 825a3f8ab1..769c927e34 100644 --- a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py +++ b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py @@ -206,3 +206,44 @@ def test_stream_chunk_matches_reference(self, tokenizer, response_parser): assert call.function is not None assert call.function.name == exp_function_name assert call.function.arguments == exp_function_arguments + + def test_stream_chunk_handles_mixed_reasoning_content_tool(self, tokenizer, response_parser): + """A single delta may contain reasoning/content/tool segments together. + + This test covers chunk shapes: + 1) ```` + 2) `` Let me think `` + 3) ``The answer is 9 OK. The`` + 4) ``fine. \\n\\n `` + """ + + def _call(delta_text: str): + ids = self._encode_ids(tokenizer, delta_text) + return response_parser.stream_chunk(delta_text=delta_text, delta_token_ids=ids) + + # 1) tag-only chunk should be swallowed + delta_msg, tool_emitted = _call('') + assert delta_msg is None + assert tool_emitted is False + + # 2) open-think plus reasoning text should emit only reasoning + delta_msg, tool_emitted = _call(' Let me think ') + assert delta_msg is not None + assert delta_msg.reasoning_content == ' Let me think ' + assert delta_msg.content is None + assert tool_emitted is False + + # 3) chunk carries reasoning end + normal content + delta_msg, tool_emitted = _call('The answer is 9 OK. The') + assert delta_msg is not None + assert delta_msg.reasoning_content == 'The answer is 9 ' + assert delta_msg.content == ' OK. The' + assert tool_emitted is False + + # 4) chunk carries stray think-close + content + tool-open + delta_msg, tool_emitted = _call('fine. \n\n ') + assert delta_msg is not None + # Stray closing tag after reasoning has ended is treated as plain content. + assert delta_msg.reasoning_content is None + assert delta_msg.content == 'fine. \n\n ' + assert tool_emitted is False From 92eb62c01daded20a60cd7907dc2524472dcec76 Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 09:55:25 +0000 Subject: [PATCH 09/14] the 4-th version --- .../deepseek_v3_reasoning_parser.py | 25 +- .../gpt_oss_reasoning_parser.py | 21 - .../identity_reasoning_parser.py | 23 +- .../reasoning_parser/qwen_reasoning_parser.py | 49 -- .../reasoning_parser/reasoning_parser.py | 135 ------ lmdeploy/serve/openai/response_parser.py | 437 ++++++----------- .../tool_parser/internlm2_tool_parser.py | 180 +------ .../openai/tool_parser/llama3_tool_parser.py | 198 +------- .../openai/tool_parser/qwen2d5_tool_parser.py | 179 +------ .../openai/tool_parser/qwen3_tool_parser.py | 153 +----- .../tool_parser/qwen3coder_tool_parser.py | 197 +++----- .../serve/openai/tool_parser/tool_parser.py | 253 +++++++--- .../server/parsers/test_qwen_parsers.py | 152 +++++- .../test_deepseek_reasoning_parser.py | 129 ----- .../test_harmony_gpt_oss_parser.py | 328 ------------- .../test_qwen_reasoning_parser.py | 266 ----------- .../server/tool_parsers/test_qwen3_parser.py | 441 ------------------ .../tool_parsers/test_qwen3coder_parser.py | 410 ---------------- 18 files changed, 589 insertions(+), 2987 deletions(-) delete mode 100644 tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py delete mode 100644 tests/test_lmdeploy/server/reasoning_parsers/test_harmony_gpt_oss_parser.py delete mode 100644 tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py delete mode 100644 tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py delete mode 100644 tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py index 513dc417aa..212a4d59a9 100644 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py @@ -2,14 +2,11 @@ from typing import TYPE_CHECKING -from lmdeploy.serve.openai.protocol import DeltaMessage -from lmdeploy.serve.openai.response_parser import StreamBuffer - from .identity_reasoning_parser import IdentityReasoningParser from .reasoning_parser import ReasoningParser if TYPE_CHECKING: - from lmdeploy.serve.openai.protocol import ChatCompletionRequest + pass class DeepSeekV3ReasoningParser(ReasoningParser): """The reasoning behavior of the DeepSeek V3.1 model varies depending on @@ -32,26 +29,6 @@ def __init__(self, tokenizer: object, **kwargs): else: self._parser = IdentityReasoningParser(tokenizer, **kwargs) - def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest') -> tuple[str | None, str | None]: - return self._parser.extract_reasoning(model_output, request) - - def extract_reasoning_streaming( - self, - delta_text: str, - delta_token_ids: list[int], - request: object, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - return self._parser.extract_reasoning_streaming( - delta_text, - delta_token_ids, - request, - stream_buffer=stream_buffer, - **kwargs, - ) - def get_reasoning_open_tag(self) -> str | None: return self._parser.get_reasoning_open_tag() diff --git a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py index 856cf3c27c..c43b7b1993 100644 --- a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py @@ -6,7 +6,6 @@ from openai_harmony import HarmonyEncodingName, Role, StreamableParser, load_harmony_encoding from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, ChatMessage, DeltaFunctionCall, DeltaMessage, @@ -14,7 +13,6 @@ FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.response_parser import StreamBuffer from .reasoning_parser import ReasoningParser, ReasoningParserManager @@ -124,25 +122,6 @@ def parse_full(self, tokens: list[int]) -> ChatMessage: :class:`~lmdeploy.serve.openai.protocol.ChatMessage`.""" return self._chat.parse_full(tokens) - def extract_reasoning_streaming( - self, - delta_text: str, - delta_token_ids: list[int], - request: object, - *, - stream_buffer: StreamBuffer, - **kwargs, - ): - """Not used; GPT-OSS uses :meth:`parse_streaming` on token ids in the - API server.""" - return None - - def extract_reasoning(self, model_output: str, request: - ChatCompletionRequest, **kwargs) -> tuple[str | None, str | None]: - """Not used for Harmony decoding; non-streaming path uses - :meth:`parse_full` on token ids.""" - return None, model_output - def get_reasoning_open_tag(self) -> str | None: return None diff --git a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py index 076a4a95ea..7ec8f65efc 100644 --- a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py @@ -3,12 +3,10 @@ # modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/identity_reasoning_parser.py from typing import TYPE_CHECKING -from lmdeploy.serve.openai.protocol import DeltaMessage from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser -from lmdeploy.serve.openai.response_parser import StreamBuffer if TYPE_CHECKING: - from lmdeploy.serve.openai.protocol import ChatCompletionRequest + pass class IdentityReasoningParser(ReasoningParser): @@ -22,25 +20,6 @@ def __init__(self, tokenizer, **kwargs): super().__init__(tokenizer, **kwargs) - def extract_reasoning_streaming( - self, - delta_text: str, - delta_token_ids: list[int], - request: object, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - # Just wrap delta_text as content, ignore reasoning - if delta_text: - return DeltaMessage(content=delta_text) - return None - - def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest') -> tuple[str | None, str | None]: - # No reasoning separation: return None for reasoning, - # and full model_output as content - return None, model_output - def get_reasoning_open_tag(self) -> str | None: return None diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py index 88f58852d6..ab76e877bb 100644 --- a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py @@ -1,14 +1,7 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/qwen3_reasoning_parser.py -from typing import TYPE_CHECKING - -from lmdeploy.serve.openai.protocol import DeltaMessage -from lmdeploy.serve.openai.response_parser import StreamBuffer - from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser -if TYPE_CHECKING: - pass @ReasoningParserManager.register_module(name=['qwen-qwq', 'qwen3', 'intern-s1', 'deepseeek-r1']) class QwenReasoningParser(ThinkingReasoningParser): @@ -24,45 +17,3 @@ class QwenReasoningParser(ThinkingReasoningParser): start_token = '' end_token = '' - - def extract_reasoning_streaming( - self, - delta_text: str, - delta_token_ids: list[int], - request: object, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - previous_token_ids = stream_buffer.previous_token_ids - # Strip from delta if present (old template / edge case where the model generates itself). - if self.start_token_id in delta_token_ids: - start_idx = delta_text.find(self.start_token) - if start_idx >= 0: - delta_text = delta_text[start_idx + len(self.start_token) :] - - if self.end_token_id in delta_token_ids: - # End token in this delta: split reasoning from content. - end_index = delta_text.find(self.end_token) - if end_index >= 0: - reasoning = delta_text[:end_index] - content = delta_text[end_index + len(self.end_token) :] - if not reasoning and not content: - return None - return DeltaMessage( - reasoning_content=reasoning if reasoning else None, - content=content if content else None, - ) - # end_token_id in IDs but not in text (already stripped) - return None - - # No end token in this delta. - if not delta_text: - # Nothing left after stripping start token. - return None - elif self.end_token_id in previous_token_ids: - # End token already passed: everything is content now. - return DeltaMessage(content=delta_text) - else: - # No end token yet: still in reasoning phase. - return DeltaMessage(reasoning_content=delta_text) diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index f62ae1fe85..cbcb769033 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -4,7 +4,6 @@ from mmengine import Registry -from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage from lmdeploy.serve.openai.response_parser import StreamBuffer ReasoningParserManager = Registry('reasoning_parser', locations=['lmdeploy.serve.openai.reasoning_parser']) @@ -24,51 +23,6 @@ def vocab(self) -> dict[str, int]: # whereas all tokenizers have .get_vocab() return self.model_tokenizer.get_vocab() - def extract_reasoning_streaming( - self, - delta_text: str, - delta_token_ids: list[int], - request: object, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - """Instance method that should be implemented for extracting reasoning - from an incomplete response; for use when handling reasoning calls and - streaming. - - Args: - delta_text: The new text chunk (may have been modified by the tool - parser before being passed here). - delta_token_ids: The new token ids for this chunk. - request: The request object. - stream_buffer: Cumulative decoding state (``ResponseParser.stream``); - Token ids from prior chunks are in ``stream_buffer.previous_token_ids`` - at the time this method runs (after ``stream_buffer.update`` for this chunk). - - Returns a DeltaMessage with reasoning_content and/or content fields, - or None if the delta should be skipped. - """ - raise NotImplementedError('ReasoningParser.extract_reasoning_streaming ' - 'has not been implemented!') - - def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', - **kwargs) -> tuple[str | None, str | None]: - """Extract reasoning content from a complete model-generated string. - - Used for non-streaming responses where we have the entire model response - available before sending to the client. - - Args: - model_output: The model-generated string to extract reasoning content from. - request: The request object that was used to generate the model_output. - - Returns: - A tuple of (reasoning_content, final_output). Either may be None. - """ - raise NotImplementedError('ReasoningParser.extract_reasoning ' - 'has not been implemented!') - def get_reasoning_open_tag(self) -> str | None: """Return reasoning opening tag string, or None if no opening tag.""" raise NotImplementedError('ReasoningParser.get_reasoning_open_tag has not been implemented!') @@ -109,95 +63,6 @@ def __init__(self, tokenizer: object, **kwargs): self.start_token_id: int = self.vocab.get(self.start_token) self.end_token_id: int = self.vocab.get(self.end_token) - def extract_reasoning_streaming( - self, - delta_text: str, - delta_token_ids: list[int], - request: object, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - """Extract reasoning content from a streaming model-generated string. - - Args: - delta_text: The new text chunk (may have been modified by the tool - parser before being passed here). - delta_token_ids: The new token ids for this chunk. - request: The request object. - stream_buffer: Cumulative decoding state (see base class). - - Returns a DeltaMessage with reasoning_content and/or content fields, - or None if the delta should be skipped. - """ - previous_token_ids = stream_buffer.previous_token_ids - - # Handle single special tokens - if len(delta_token_ids) == 1 and (delta_token_ids[0] in [self.start_token_id, self.end_token_id]): - return None - - # Check if start tag is in previous tokens - if self.start_token_id in previous_token_ids: - if self.end_token_id in delta_token_ids: - # Both start and end in delta -> extract between them - end_idx = delta_text.find(self.end_token) - reasoning_content = delta_text[:end_idx] - content = delta_text[end_idx + len(self.end_token):] - return DeltaMessage(reasoning_content=reasoning_content, content=content if content else None) - elif self.end_token_id in previous_token_ids: - # end in previous, no start -> reasoning is done - return DeltaMessage(content=delta_text) - else: - # start in previous, no end -> reasoning continues - return DeltaMessage(reasoning_content=delta_text) - elif self.start_token_id in delta_token_ids: - start_index = delta_text.find(self.start_token) - if self.end_token_id in delta_token_ids: - # Both start and end in delta -> extract between them - end_index = delta_text.find(self.end_token) - reasoning_content = delta_text[start_index + len(self.start_token) : end_index] - content = delta_text[end_index + len(self.end_token) :] - return DeltaMessage( - reasoning_content=reasoning_content, content=content if content else None - ) - else: - # start token in delta, no end token in delta, reasoning content continues - return DeltaMessage(reasoning_content=delta_text[start_index + len(self.start_token):]) - else: - # not find thinking start token - return DeltaMessage(content=delta_text) - - def extract_reasoning(self, model_output: str, request: 'ChatCompletionRequest', **kwargs) -> tuple[str, str]: - """Extract reasoning content from a complete model-generated string. - - Args: - model_output: The model-generated string to extract reasoning content from. - request: The request object that was used to generate the model_output. - - Returns: - A tuple of (reasoning_content, final_output). Either may be None. - """ - - if self.start_token not in model_output and self.end_token not in model_output: - return None, model_output - - model_output_parts = model_output.partition(self.start_token) - model_output = ( - model_output_parts[2] if model_output_parts[1] else model_output_parts[0] - ) - - # For models that may not generate start token, - # assume the reasoning content is always at the start. - if self.end_token not in model_output: - return model_output, None - else: - reasoning, _, content = model_output.partition(self.end_token) - # If generation stops right after end-of-think, return None content - final_content = content or None - # If the model_output is like "...", return None reasoning - reasoning = reasoning or None - return reasoning, final_content - def get_reasoning_open_tag(self) -> str | None: return self.start_token diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py index c05d1e0a05..b97a79a3f8 100644 --- a/lmdeploy/serve/openai/response_parser.py +++ b/lmdeploy/serve/openai/response_parser.py @@ -2,21 +2,15 @@ """Unified profile-driven streaming parser for reasoning/content/tool calls.""" from __future__ import annotations -import json from dataclasses import dataclass, field from typing import TYPE_CHECKING, ClassVar -import partial_json_parser -import shortuuid -from partial_json_parser.core.options import Allow from transformers import PreTrainedTokenizerBase from lmdeploy.serve.openai.protocol import ( ChatCompletionRequest, - DeltaFunctionCall, DeltaMessage, DeltaToolCall, - FunctionCall, ToolCall, ) from lmdeploy.utils import get_logger @@ -58,19 +52,23 @@ class ProtocolProfile: @dataclass -class _ToolDecodeState: - active_tool_id: str = '' - active_tool_index: int = -1 - name_emitted: bool = False - args_emitted_len: int = 0 - prev_args_json: str | None = None - args_prefix_emitted: bool = False - value_chars_emitted: int = 0 - args_closed_emitted: bool = False +class _QueuedDelta: + delta: DeltaMessage + tool_calls_emitted: bool = False class ResponseParser: - """Single entry for streaming and complete parsing.""" + """Unified parser for streaming and complete assistant responses. + + It separates model output into: + - plain assistant content + - reasoning content + - tool-call deltas + + Parsing is protocol/profile-driven and supports mixed chunks where one + ``delta_text`` may contain multiple segments (for example reasoning close + plus plain text plus tool open tag). + """ reasoning_parser_cls: ClassVar[type[ReasoningParser] | None] = None tool_parser_cls: ClassVar[type[ToolParser] | None] = None @@ -80,8 +78,12 @@ class ResponseParser: @classmethod def chat_template_kwargs_from_request(cls, request: ChatCompletionRequest) -> dict: - """Merge ``request.enable_thinking`` into ``chat_template_kwargs`` - (deprecated field path).""" + """Normalize parser-related template kwargs from the request. + + ``enable_thinking`` is a deprecated top-level field. This helper maps + it into ``chat_template_kwargs`` so downstream parser behavior can rely + on one normalized source. + """ chat_template_kwargs = request.chat_template_kwargs or {} if request.enable_thinking is not None: logger.warning('`enable_thinking` will be deprecated in the future, ' @@ -115,13 +117,13 @@ def __init__( self.stream_buffer = StreamBuffer() self.profile = self._build_profile() - if (self.reasoning_parser is not None and self.enable_thinking is not False): + if (self.reasoning_parser is not None and self.enable_thinking is not False + and self.profile.starts_in_reasoning_mode): self._mode = self.MODE_REASONING else: self._mode = self.MODE_PLAIN self._pending = '' - self._tool_payload = '' - self._tool_decode_state = _ToolDecodeState() + self._queued_deltas: list[_QueuedDelta] = [] def _stream_update(self, delta_text: str, delta_token_ids: list[int]) -> None: self.stream_buffer.update(delta_text, delta_token_ids) @@ -135,7 +137,18 @@ def stream_chunk( delta_token_ids: list[int], **kwargs, ) -> tuple[DeltaMessage | None, bool]: - """Parse a single streamed chunk.""" + """Parse one streamed chunk into delta message channels. + + Args: + delta_text: New text fragment produced in this stream step. + delta_token_ids: Token ids corresponding to ``delta_text``. + + Returns: + ``(delta_message, tool_calls_emitted)`` where: + - ``delta_message`` is ``None`` when this step has no visible delta. + - ``tool_calls_emitted`` is ``True`` if at least one tool-call + delta is emitted in this step. + """ # Special-case: some backends emit a leading empty delta (no text, no # tokens) before any actual content. Tests treat this as a visible empty # content delta. @@ -152,70 +165,69 @@ def stream_chunk( self._stream_update(delta_text, delta_token_ids) self._pending += delta_text - - content_parts: list[str] = [] - reasoning_parts: list[str] = [] - tool_calls: list[DeltaToolCall] = [] - tool_calls_emitted = False + produced_any = False while True: progressed = False if self._mode == self.MODE_PLAIN: emitted, progressed = self._consume_plain() if emitted: - content_parts.append(emitted) + self._queued_deltas.append(_QueuedDelta(DeltaMessage(role='assistant', content=emitted), False)) + produced_any = True elif self._mode == self.MODE_REASONING: emitted, progressed = self._consume_reasoning() if emitted: if self.enable_thinking is False: - content_parts.append(emitted) + self._queued_deltas.append(_QueuedDelta(DeltaMessage(role='assistant', content=emitted), False)) else: - reasoning_parts.append(emitted) - else: # self.MODE_TOOL + self._queued_deltas.append( + _QueuedDelta(DeltaMessage(role='assistant', reasoning_content=emitted), False)) + produced_any = True + if self._mode == self.MODE_TOOL: + # self._consume_plain() might change the mode to MODE_TOOL + # so we need to check the mode again new_calls, progressed = self._consume_tool() if new_calls: - tool_calls.extend(new_calls) - tool_calls_emitted = True + self._queued_deltas.append( + _QueuedDelta(DeltaMessage(role='assistant', tool_calls=new_calls), True)) + produced_any = True if not progressed: break - delta_message = DeltaMessage(role='assistant') - if content_parts: - delta_message.content = ''.join(content_parts) - if reasoning_parts: - delta_message.reasoning_content = ''.join(reasoning_parts) - if tool_calls: - delta_message.tool_calls = tool_calls - # 5. Special case: a trailing empty delta (delta_text == '') after non-empty # output should be surfaced as an explicit empty content delta so that # streaming clients see the final "no-op" chunk (some backends do this). - emitted_trailing_empty = False if ( delta_text == '' - and delta_message.content is None - and delta_message.reasoning_content is None - and not delta_message.tool_calls + and not produced_any and self.stream_buffer.current_text != '' ): - delta_message.content = '' - emitted_trailing_empty = True + self._queued_deltas.append(_QueuedDelta(DeltaMessage(role='assistant', content=''), False)) self._stream_step() - - # 6. If there is no reasoning, no tool_calls, and no visible content - # change, treat this chunk as a non-delta. - if ( - delta_message.reasoning_content is None - and not delta_message.tool_calls - and (delta_message.content is None or delta_message.content == '') - and not emitted_trailing_empty - ): - return None, tool_calls_emitted - - return delta_message, tool_calls_emitted + if not self._queued_deltas: + return None, False + queued = self._queued_deltas.pop(0) + return queued.delta, queued.tool_calls_emitted def _consume_plain(self) -> tuple[str | None, bool]: + """Consume buffered text while in plain mode. + + Behavior: + - Finds the earliest protocol opening tag (reasoning/tool) in + ``self._pending``. + - If no full tag is present, emits only the safe plain-text prefix and + preserves possible partial-tag suffix for the next chunk. + - If a tag is found, emits text before the tag as plain content, + consumes the tag, and switches mode: + - reasoning open tag -> ``MODE_REASONING`` + - tool open tag -> ``MODE_TOOL`` (also initializes tool-call state) + + Returns: + ``(emitted_text, progressed)`` where ``emitted_text`` is the plain + content produced in this step (or ``None``), and ``progressed`` + indicates whether parser state/input was consumed. + """ tags = [t for t in (self.profile.reasoning_open_tag, self.profile.tool_open_tag) if t] if not tags: if not self._pending: @@ -224,6 +236,7 @@ def _consume_plain(self) -> tuple[str | None, bool]: self._pending = '' return out, True + # Find the earliest protocol open tag. earliest_idx = -1 earliest_tag = None for tag in tags: @@ -232,12 +245,13 @@ def _consume_plain(self) -> tuple[str | None, bool]: earliest_idx = idx earliest_tag = tag + # No protocol open tag found, treat the whole pending text as plain content. if earliest_idx < 0: - emit, remain = self._split_on_partial_prefix(self._pending, tags) - if emit == '': + if not self._pending: return None, False - self._pending = remain - return emit, True + out = self._pending + self._pending = '' + return out, True # Emit content before protocol open tag. prefix = self._pending[:earliest_idx] @@ -246,239 +260,98 @@ def _consume_plain(self) -> tuple[str | None, bool]: self._mode = self.MODE_REASONING else: self._mode = self.MODE_TOOL - self._tool_payload = '' - self._start_tool_call() + if self.tool_parser is not None: + self.tool_parser.start_tool_call() return (prefix if prefix else None), True def _consume_reasoning(self) -> tuple[str | None, bool]: - # Drop explicit open tag if model emits it. + """Consume buffered text while in reasoning mode. + + Behavior: + - Drops the explicit open tag if model emits it. + - If no close tag is present, emits only the safe reasoning-text prefix and + preserves possible partial-tag suffix for the next chunk. + - If a close tag is found, emits text before the close tag as reasoning content, + consumes the close tag, and switches mode to ``MODE_PLAIN``. + + Returns: + ``(emitted_text, progressed)`` where ``emitted_text`` is the reasoning + content produced in this step (or ``None``), and ``progressed`` + indicates whether parser state/input was consumed. + """ + open_tag = self.profile.reasoning_open_tag + # Drop explicit open tag if model emits it. if open_tag and self._pending.startswith(open_tag): self._pending = self._pending[len(open_tag):] return None, True close_tag = self.profile.reasoning_close_tag if not close_tag: + raise RuntimeError('Invariant violated: MODE_REASONING requires a reasoning_close_tag.') + + idx = self._pending.find(close_tag) + # No close tag found, treat the whole pending text as reasoning content. + if idx < 0: if not self._pending: return None, False out = self._pending self._pending = '' return out, True - earliest_idx = self._pending.find(close_tag) - - if earliest_idx < 0: - emit, remain = self._split_on_partial_prefix(self._pending, [close_tag]) - if emit == '': - return None, False - self._pending = remain - return emit, True - - reasoning_chunk = self._pending[:earliest_idx] - self._pending = self._pending[earliest_idx + len(close_tag):] + reasoning_chunk = self._pending[:idx] + self._pending = self._pending[idx + len(close_tag):] + # reasoning part is done, switch to plain mode self._mode = self.MODE_PLAIN return (reasoning_chunk if reasoning_chunk else None), True def _consume_tool(self) -> tuple[list[DeltaToolCall], bool]: + """Consume buffered text while in tool mode. + + Behavior: + - Treats ``self._pending`` as tool payload bytes until ``tool_close_tag`` + is found. + - For non-final payload chunks, forwards text to + ``tool_parser.decode_tool_incremental(..., final=False)``. + - For the final payload chunk (before close tag), forwards text with + ``final=True``, then calls ``tool_parser.finish_tool_call()`` and + switches mode back to ``MODE_PLAIN``. + - This method is format-agnostic: JSON/XML/other details are handled + entirely by the concrete tool parser implementation. + + Returns: + ``(tool_call_deltas, progressed)`` where ``tool_call_deltas`` is the + list emitted by the tool parser for this step (possibly empty), and + ``progressed`` indicates whether parser state/input was consumed. + """ + if self.tool_parser is None: + raise RuntimeError('Invariant violated: MODE_TOOL requires a tool_parser.') + close_tag = self.profile.tool_close_tag if not close_tag: if not self._pending: return [], False emit = self._pending self._pending = '' - self._tool_payload += emit - return self._decode_tool_incremental(added_text=emit, final=False), True + return self.tool_parser.decode_tool_incremental(added_text=emit, final=False), True - earliest_idx = self._pending.find(close_tag) + idx = self._pending.find(close_tag) - if earliest_idx < 0: - emit, remain = self._split_on_partial_prefix(self._pending, [close_tag]) - if emit == '': + if idx < 0: + if not self._pending: return [], False - self._pending = remain - self._tool_payload += emit - return self._decode_tool_incremental(added_text=emit, final=False), True + emit = self._pending + self._pending = '' + return self.tool_parser.decode_tool_incremental(added_text=emit, final=False), True # Final chunk inside tool block. - inner = self._pending[:earliest_idx] - self._tool_payload += inner - self._pending = self._pending[earliest_idx + len(close_tag):] - calls = self._decode_tool_incremental(added_text=inner, final=True) - self._finish_tool_call() + inner = self._pending[:idx] + self._pending = self._pending[idx + len(close_tag):] + calls = self.tool_parser.decode_tool_incremental(added_text=inner, final=True) + self.tool_parser.finish_tool_call() self._mode = self.MODE_PLAIN return calls, True - def _start_tool_call(self) -> None: - st = self._tool_decode_state - st.active_tool_index += 1 - st.active_tool_id = f'chatcmpl-tool-{shortuuid.random()}' - st.name_emitted = False - st.args_emitted_len = 0 - st.args_prefix_emitted = False - st.value_chars_emitted = 0 - st.args_closed_emitted = False - - def _finish_tool_call(self) -> None: - st = self._tool_decode_state - st.active_tool_id = '' - st.name_emitted = False - st.args_emitted_len = 0 - st.prev_args_json = None - st.args_prefix_emitted = False - st.value_chars_emitted = 0 - st.args_closed_emitted = False - self._tool_payload = '' - - def _decode_tool_incremental(self, added_text: str, final: bool) -> list[DeltaToolCall]: - if self.profile.tool_payload_format != 'json': - return [] - payload = self._tool_payload.strip() - if not payload: - return [] - - st = self._tool_decode_state - flags = Allow.ALL if st.name_emitted else Allow.ALL & ~Allow.STR - try: - obj = partial_json_parser.loads(payload, flags) - except partial_json_parser.core.exceptions.MalformedJSON: - return [] - - if not isinstance(obj, dict): - return [] - - out: list[DeltaToolCall] = [] - if not st.name_emitted: - fn_name = obj.get('name') - if isinstance(fn_name, str) and fn_name: - out.append( - DeltaToolCall( - id=st.active_tool_id, - index=st.active_tool_index, - type='function', - function=DeltaFunctionCall(name=fn_name), - )) - st.name_emitted = True - - args_obj = obj.get('arguments', obj.get('parameters', None)) - if args_obj is not None: - # Value-stream mode for dict-with-string-values arguments. This - # matches the reference chunk contract: emit object open once, then - # only value text deltas, then close quote+brace at finalization. - if isinstance(args_obj, dict): - items = list(args_obj.items()) - if not st.args_prefix_emitted and items: - first_key = items[0][0] - out.append( - DeltaToolCall( - id=st.active_tool_id, - index=st.active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=f'{{\"{first_key}\": \"')), - ) - st.args_prefix_emitted = True - - values_concat = ''.join(v for _, v in items if isinstance(v, str)) - if len(values_concat) > st.value_chars_emitted: - diff = values_concat[st.value_chars_emitted:] - out.append( - DeltaToolCall( - id=st.active_tool_id, - index=st.active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=diff), - )) - st.value_chars_emitted = len(values_concat) - - if self._is_complete_json(payload) and st.args_prefix_emitted and not st.args_closed_emitted: - out.append( - DeltaToolCall( - id=st.active_tool_id, - index=st.active_tool_index, - type=None, - function=DeltaFunctionCall(arguments='"}'), - )) - st.args_closed_emitted = True - return out - - args_json = json.dumps(args_obj, ensure_ascii=False) - # Do not emit/track empty dict/list placeholders during partial decode. - if args_json not in ('{}', '[]'): - emitted_arg = False - candidate: str | None = None - if self._is_complete_json(payload): - candidate = args_json - elif st.prev_args_json: - candidate = self._common_prefix(st.prev_args_json, args_json) - elif st.args_emitted_len == 0 and added_text: - pos = args_json.find(added_text) - if pos >= 0: - candidate = args_json[:pos + len(added_text)] - - if candidate and len(candidate) > st.args_emitted_len: - diff = candidate[st.args_emitted_len:] - if final or any(ch.isalnum() for ch in diff): - out.append( - DeltaToolCall( - id=st.active_tool_id, - index=st.active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=diff), - )) - st.args_emitted_len = len(candidate) - emitted_arg = True - - # Some partial decodes don't advance parsed JSON although text - # has advanced (e.g., unfinished string body). Stream lexical - # text for content-bearing chunks to keep deltas monotonic. - if ( - not emitted_arg - and st.args_emitted_len > 0 - and added_text - and any(ord(ch) > 127 for ch in added_text) - ): - out.append( - DeltaToolCall( - id=st.active_tool_id, - index=st.active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=added_text), - )) - st.args_emitted_len += len(added_text) - st.prev_args_json = args_json - return out - - @staticmethod - def _is_complete_json(text: str) -> bool: - try: - json.loads(text) - return True - except json.JSONDecodeError: - return False - - @staticmethod - def _common_prefix(s1: str, s2: str) -> str: - i = 0 - n = min(len(s1), len(s2)) - while i < n and s1[i] == s2[i]: - i += 1 - return s1[:i] - - @staticmethod - def _split_on_partial_prefix(text: str, tags: list[str]) -> tuple[str, str]: - """Split text into (emit, remain) while preserving possible partial - tags.""" - if not text: - return '', '' - max_keep = 0 - upper = min(len(text), max((len(t) for t in tags), default=0) - 1) - for k in range(1, upper + 1): - suffix = text[-k:] - if any(tag.startswith(suffix) for tag in tags): - max_keep = k - if max_keep == 0: - return text, '' - return text[:-max_keep], text[-max_keep:] - def _build_profile(self) -> ProtocolProfile: profile = ProtocolProfile(starts_in_reasoning_mode=False) rparser = self.reasoning_parser @@ -488,12 +361,17 @@ def _build_profile(self) -> ProtocolProfile: profile.reasoning_open_tag = rparser.get_reasoning_open_tag() profile.reasoning_close_tag = rparser.get_reasoning_close_tag() profile.starts_in_reasoning_mode = bool(rparser.starts_in_reasoning_mode()) + if not profile.reasoning_close_tag: + raise ValueError(f'Reasoning parser {rparser.__class__.__name__} must provide a reasoning close tag') if tparser is not None and self.request.tool_choice != 'none': profile.tool_open_tag = tparser.get_tool_open_tag() profile.tool_close_tag = tparser.get_tool_close_tag() profile.tool_payload_format = tparser.get_tool_payload_format() - + if not profile.tool_open_tag: + raise ValueError(f'Tool parser {tparser.__class__.__name__} must provide a tool open tag') + if not profile.tool_close_tag: + raise ValueError(f'Tool parser {tparser.__class__.__name__} must provide a tool close tag') return profile def parse_complete( @@ -501,8 +379,17 @@ def parse_complete( text: str, **kwargs, ) -> tuple[str, list | None, str | None]: - """Non-streaming parse with the same profile-driven protocol - semantics.""" + """Parse the final non-streaming text output. + + Args: + text: Full generated output text. + + Returns: + A tuple ``(content, tool_calls, reasoning_content)``: + - ``content``: plain assistant-visible text, or ``None`` + - ``tool_calls``: parsed tool calls, or ``None`` + - ``reasoning_content``: separated reasoning text, or ``None`` + """ content_parts: list[str] = [] reasoning_parts: list[str] = [] tool_calls: list[ToolCall] = [] @@ -557,7 +444,7 @@ def parse_complete( content_parts.append(text[open_idx:]) break tool_payload = text[open_idx + len(open_tag):close_idx].strip() - parsed_call = self._parse_tool_call_complete(tool_payload) + parsed_call = self.tool_parser.parse_tool_call_complete(tool_payload) if self.tool_parser else None if parsed_call is not None: tool_calls.append(parsed_call) pos = close_idx + len(close_tag) @@ -576,21 +463,3 @@ def _find_first(text: str, tags: list[str], start: int) -> tuple[int, str]: best_idx = idx best_tag = tag return best_idx, best_tag - - def _parse_tool_call_complete(self, payload: str) -> ToolCall | None: - if self.profile.tool_payload_format != 'json': - return None - if not payload: - return None - try: - obj = json.loads(payload) - except json.JSONDecodeError: - return None - if not isinstance(obj, dict): - return None - name = obj.get('name') - if not isinstance(name, str) or not name: - return None - args_obj = obj.get('arguments', obj.get('parameters', {})) - args_json = json.dumps(args_obj, ensure_ascii=False) - return ToolCall(function=FunctionCall(name=name, arguments=args_json)) diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index 429a0f6c4e..5b804d5518 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -1,25 +1,14 @@ # Copyright (c) OpenMMLab. All rights reserved. -import json -from collections.abc import Sequence -import partial_json_parser -import shortuuid -from partial_json_parser.core.options import Allow from lmdeploy.serve.openai.protocol import ( ChatCompletionRequest, - DeltaFunctionCall, - DeltaMessage, DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -from .utils import extract_intermediate_diff logger = get_logger('lmdeploy') @@ -59,167 +48,10 @@ def get_tool_close_tag(self) -> str | None: def get_tool_payload_format(self) -> str: return 'json' - def detect_tool_start_tag( - self, - delta_text: str, - delta_token_ids: Sequence[int], - *, - stream_buffer: StreamBuffer, - request: ChatCompletionRequest, - ) -> int | None: - """Return index where InternLM action block starts in - ``delta_text``.""" - text = stream_buffer.current_text - start_idx = text.rfind('<|action_start|><|plugin|>') - end_idx = text.rfind('<|action_end|>') - if start_idx >= 0 and end_idx < start_idx: - return 0 - plugin_start = '<|action_start|><|plugin|>\n' - idx = delta_text.find(plugin_start) - if idx >= 0: - return idx - fallback = '<|action_start|><|plugin|>' - idx = delta_text.find(fallback) - return idx if idx >= 0 else None + def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + """InternLM2 tool payload is JSON; reuse shared JSON incremental + decoder.""" + return self._decode_tool_incremental_json(added_text=added_text, final=final) - def extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - current_text = stream_buffer.current_text - if '<|action_start|>' not in current_text: - self.parse_cursor = len(current_text) - return DeltaMessage(content=delta_text) - # if the tool call is sended, return a empty delta message - # to make sure the finish_reason will be send correctly. - if self.current_tool_id > 0: - return DeltaMessage(content='') - - last_pos = self.parse_cursor - if '<|action_start|><|plugin|>\n' not in current_text[last_pos:]: - return None - - new_delta = current_text[last_pos:] - text, action = new_delta.split('<|action_start|><|plugin|>\n') - - if len(text) > 0: - self.parse_cursor = self.parse_cursor + len(text) - return DeltaMessage(content=text) - - action = action.strip() - action = action.split('<|action_end|>'.strip())[0] - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent \ - else Allow.ALL & ~Allow.STR - - try: - parsable_arr = action - - # tool calls are generated in an object in inernlm2 - # it's not support parallel tool calls - try: - tool_call_arr: dict = partial_json_parser.loads(parsable_arr, flags) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug('not enough tokens to parse into JSON yet') - return None - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - if not self.current_tool_name_sent: - function_name = tool_call_arr.get('name') - if function_name: - self.current_tool_id = self.current_tool_id + 1 - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - type='function', - id=f'chatcmpl-tool-{shortuuid.random()}', - function=DeltaFunctionCall(name=function_name).model_dump(exclude_none=True)) - ]) - self.current_tool_name_sent = True - self.streamed_args_for_tool.append('') - else: - delta = None - # now we know we're on the same tool call and we're streaming - # arguments - else: - prev_arguments = self.get_argments(self.prev_tool_call_arr[self.current_tool_id]) - cur_arguments = self.get_argments(tool_call_arr) - - # not arguments generated - if not cur_arguments and not prev_arguments: - delta = None - # will never happen - elif not cur_arguments and prev_arguments: - logger.error('INVARIANT - impossible to have arguments reset ' - 'mid-arguments') - delta = None - # first time to get parameters - elif cur_arguments and not prev_arguments: - cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False) - - arguments_delta = cur_arguments_json[:cur_arguments_json.index(delta_text) + len(delta_text)] - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - function=DeltaFunctionCall(arguments=arguments_delta).model_dump( - exclude_none=True)) - ]) - self.streamed_args_for_tool[self.current_tool_id] += arguments_delta - # both prev and cur parameters, send the increase parameters - elif cur_arguments and prev_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - - argument_diff = extract_intermediate_diff(cur_args_json, prev_args_json) - - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - function=DeltaFunctionCall(arguments=argument_diff).model_dump(exclude_none=True)) - ]) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - - # check to see if the name is defined and has been sent. if so, - # stream the name - otherwise keep waiting - # finish by setting old and returning None as base case - tool_call_arr['arguments'] = self.get_argments(tool_call_arr) - self.prev_tool_call_arr = [tool_call_arr] - return delta - except Exception: - logger.exception('Error trying to handle streaming tool call.') - logger.debug('Skipping chunk as a result of tool streaming extraction ' - 'error') - return None - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - text = model_output - tools = request.tools - if '<|action_start|><|plugin|>' in text: - text, action = text.split('<|action_start|><|plugin|>') - action = action.split('<|action_end|>'.strip())[0] - action = action[action.find('{'):] - action_dict = json.loads(action) - name, parameters = action_dict['name'], json.dumps(action_dict.get('parameters', - action_dict.get('arguments', {})), - ensure_ascii=False) - - if not tools or name not in [t.function.name for t in tools]: - ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=text) - - tool_calls = [ToolCall(function=FunctionCall(name=name, arguments=parameters))] - return ExtractedToolCallInformation(tools_called=True, - tool_calls=tool_calls, - content=text if len(text) > 0 else None) - - return ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=text) + def parse_tool_call_complete(self, payload: str) -> ToolCall | None: + return self._parse_tool_call_complete_json(payload) diff --git a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py index 42b37eebd8..29d091fa0e 100644 --- a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py @@ -1,26 +1,13 @@ # Copyright (c) OpenMMLab. All rights reserved. -import json import re -from collections.abc import Sequence - -import partial_json_parser -import shortuuid -from partial_json_parser.core.options import Allow from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - DeltaFunctionCall, - DeltaMessage, DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -from .utils import find_common_prefix, is_complete_json, partial_json_loads logger = get_logger('lmdeploy') @@ -53,183 +40,10 @@ def get_tool_close_tag(self) -> str | None: def get_tool_payload_format(self) -> str: return 'json' - def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) -> ExtractedToolCallInformation: - """Extract the tool calls from a complete model response.""" - try: - # load the JSON, and then use it to build the Function and - # Tool Call - action, _ = model_output.split('') - parameters = action[action.find('{'):] - name = action.split('{')[0] - call_info_list = [(name, parameters)] - - tool_calls: list[ToolCall] = [ - ToolCall(type='function', function=FunctionCall(name=name, arguments=arguments)) - for name, arguments in call_info_list - ] - - # get any content before the tool call - ret = ExtractedToolCallInformation(tools_called=True, tool_calls=tool_calls, content=None) - return ret - - except Exception: - logger.exception('Error in extracting tool call from response.') - # return information to just treat the tool call as regular JSON - return ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=model_output) - - def detect_tool_start_tag( - self, - delta_text: str, - delta_token_ids: Sequence[int], - *, - stream_buffer: StreamBuffer, - request: ChatCompletionRequest, - ) -> int | None: - """Return index where Llama3 tool-call JSON protocol starts.""" - if stream_buffer.previous_text.startswith(self.bot_token) or stream_buffer.previous_text.startswith('{'): - return 0 - idx = delta_text.find(self.bot_token) - if idx >= 0: - return idx - # Llama may emit raw JSON without the python tag. - # Keep this conservative to avoid splitting ordinary prose with braces. - if stream_buffer.previous_text == '' and delta_text.startswith('{'): - return 0 - return None - - def extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - current_text = stream_buffer.current_text - if not (current_text.startswith(self.bot_token) or current_text.startswith('{')): - return DeltaMessage(content=delta_text) - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent \ - else Allow.ALL & ~Allow.STR - try: - tool_call_arr = [] - is_complete = [] - try: - # depending on the prompt format the Llama model may or may not - # prefix the output with the <|python_tag|> token - start_idx = len(self.bot_token) if current_text.startswith(self.bot_token) else 0 - while start_idx < len(current_text): - (obj, end_idx) = partial_json_loads(current_text[start_idx:], flags) - is_complete.append(is_complete_json(current_text[start_idx:start_idx + end_idx])) - start_idx += end_idx + len('; ') - # depending on the prompt Llama can use - # either arguments or parameters - if 'parameters' in obj: - assert 'arguments' not in obj, \ - 'model generated both parameters and arguments' - obj['arguments'] = obj['parameters'] - tool_call_arr.append(obj) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug('not enough tokens to parse into JSON yet') - return None - - # select as the current tool call the one we're on the state at - current_tool_call: dict = tool_call_arr[self.current_tool_id] \ - if len(tool_call_arr) > 0 else {} - - # case -- if no tokens have been streamed for the tool, e.g. - # only the array brackets, stream nothing - if len(tool_call_arr) == 0: - return None - - # case: we are starting a new tool in the array - # -> array has > 0 length AND length has moved past cursor - elif (len(tool_call_arr) > 0 and len(tool_call_arr) > self.current_tool_id + 1): - - # if we're moving on to a new call, first make sure we - # haven't missed anything in the previous one that was - # auto-generated due to JSON completions, but wasn't - # streamed to the client yet. - if self.current_tool_id >= 0: - cur_arguments = current_tool_call.get('arguments') - if cur_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - argument_diff = cur_args_json[sent:] - - logger.debug('got arguments diff: %s', argument_diff) - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - function=DeltaFunctionCall(arguments=argument_diff).model_dump( - exclude_none=True)) - ]) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - else: - delta = None - else: - delta = None - # re-set stuff pertaining to progress in the current tool - self.current_tool_id = len(tool_call_arr) - 1 - self.current_tool_name_sent = False - self.streamed_args_for_tool.append('') - logger.debug('starting on new tool %d', self.current_tool_id) - return delta - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - elif not self.current_tool_name_sent: - function_name = current_tool_call.get('name') - if function_name: - - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - type='function', - id=f'chatcmpl-tool-{shortuuid.random()}', - function=DeltaFunctionCall(name=function_name).model_dump(exclude_none=True)) - ]) - self.current_tool_name_sent = True - else: - delta = None - - # now we know we're on the same tool call and we're streaming - # arguments - else: - cur_arguments = current_tool_call.get('arguments') - delta = None - - if cur_arguments: - sent = len(self.streamed_args_for_tool[self.current_tool_id]) - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_arguments = self.prev_tool_call_arr[self.current_tool_id].get('arguments') - - argument_diff = None - if is_complete[self.current_tool_id]: - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - if cur_args_json != prev_args_json: - - prefix = find_common_prefix(prev_args_json, cur_args_json) - argument_diff = prefix[sent:] - - if argument_diff is not None: - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - function=DeltaFunctionCall(arguments=argument_diff).model_dump( - exclude_none=True)) - ]) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - - self.prev_tool_call_arr = tool_call_arr - return delta + def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + """Llama3 tool payload is JSON; reuse shared JSON incremental + decoder.""" + return self._decode_tool_incremental_json(added_text=added_text, final=final) - except Exception: - logger.exception('Error trying to handle streaming tool call.') - logger.debug('Skipping chunk as a result of tool streaming extraction ' - 'error') - return None + def parse_tool_call_complete(self, payload: str) -> ToolCall | None: + return self._parse_tool_call_complete_json(payload) diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py index 9f29e30e1b..35cbb95449 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py @@ -1,26 +1,13 @@ # Copyright (c) OpenMMLab. All rights reserved. -import json -import re -from collections.abc import Sequence -import partial_json_parser -import shortuuid -from partial_json_parser.core.options import Allow from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - DeltaFunctionCall, - DeltaMessage, DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -from .utils import extract_intermediate_diff logger = get_logger('lmdeploy') @@ -55,164 +42,10 @@ def get_tool_close_tag(self) -> str | None: def get_tool_payload_format(self) -> str: return 'json' - def detect_tool_start_tag( - self, - delta_text: str, - delta_token_ids: Sequence[int], - *, - stream_buffer: StreamBuffer, - request: ChatCompletionRequest, - ) -> int | None: - """Return index in ``delta_text`` where ```` starts.""" - text = stream_buffer.current_text - start_idx = text.rfind(self.tool_start_token) - end_idx = text.rfind(self.tool_end_token) - if start_idx >= 0 and end_idx < start_idx: - return 0 - idx = delta_text.find(self.tool_start_token) - return idx if idx >= 0 else None + def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + """Qwen2.5 tool payload is JSON; reuse shared JSON incremental + decoder.""" + return self._decode_tool_incremental_json(added_text=added_text, final=final) - def extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - current_text = stream_buffer.current_text - if self.tool_start_token not in current_text: - self.parse_cursor = len(current_text) - return DeltaMessage(content=delta_text) - # if the tool call is sended, return a empty delta message - # to make sure the finish_reason will be send correctly. - if self.current_tool_id > 0: - return DeltaMessage(content='') - - last_pos = self.parse_cursor - if self.tool_start_token not in current_text[last_pos:]: - return None - - new_delta = current_text[last_pos:] - text, action = new_delta.split(self.tool_start_token) - - if len(text) > 0: - self.parse_cursor = self.parse_cursor + len(text) - return DeltaMessage(content=text) - - action = action.strip() - action = action.split(self.tool_end_token.strip())[0] - - # bit mask flags for partial JSON parsing. If the name hasn't been - # sent yet, don't allow sending - # an incomplete string since OpenAI only ever (as far as I have - # seen) allows sending the entire tool/ function name at once. - flags = Allow.ALL if self.current_tool_name_sent \ - else Allow.ALL & ~Allow.STR - - try: - parsable_arr = action - - # tool calls are generated in an object in inernlm2 - # it's not support parallel tool calls - try: - tool_call_arr: dict = partial_json_parser.loads(parsable_arr, flags) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug('not enough tokens to parse into JSON yet') - return None - - # if the current tool name hasn't been sent, send if available - # - otherwise send nothing - if not self.current_tool_name_sent: - function_name = tool_call_arr.get('name') - if function_name: - self.current_tool_id = self.current_tool_id + 1 - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - type='function', - id=f'chatcmpl-tool-{shortuuid.random()}', - function=DeltaFunctionCall(name=function_name).model_dump(exclude_none=True)) - ]) - self.current_tool_name_sent = True - self.streamed_args_for_tool.append('') - else: - delta = None - # now we know we're on the same tool call and we're streaming - # arguments - else: - prev_arguments = self.get_argments(self.prev_tool_call_arr[self.current_tool_id]) - cur_arguments = self.get_argments(tool_call_arr) - - # not arguments generated - if not cur_arguments and not prev_arguments: - delta = None - # will never happen - elif not cur_arguments and prev_arguments: - logger.error('INVARIANT - impossible to have arguments reset ' - 'mid-arguments') - delta = None - # first time to get parameters - elif cur_arguments and not prev_arguments: - cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False) - - arguments_delta = cur_arguments_json[:cur_arguments_json.index(delta_text) + len(delta_text)] - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - function=DeltaFunctionCall(arguments=arguments_delta).model_dump( - exclude_none=True)) - ]) - self.streamed_args_for_tool[self.current_tool_id] += arguments_delta - # both prev and cur parameters, send the increase parameters - elif cur_arguments and prev_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - - argument_diff = extract_intermediate_diff(cur_args_json, prev_args_json) - - delta = DeltaMessage(tool_calls=[ - DeltaToolCall(index=self.current_tool_id, - function=DeltaFunctionCall(arguments=argument_diff).model_dump(exclude_none=True)) - ]) - self.streamed_args_for_tool[self.current_tool_id] += argument_diff - - # check to see if the name is defined and has been sent. if so, - # stream the name - otherwise keep waiting - # finish by setting old and returning None as base case - tool_call_arr['arguments'] = self.get_argments(tool_call_arr) - self.prev_tool_call_arr = [tool_call_arr] - return delta - except Exception: - logger.exception('Error trying to handle streaming tool call.') - logger.debug('Skipping chunk as a result of tool streaming extraction ' - 'error') - return None - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - text = model_output - if self.tool_start_token in text: - - # get tool_call in text - match_result_list = re.findall(self.pattern, text, re.DOTALL) - tool_calls = [] - for match_result in match_result_list: - action = json.loads(match_result) - name, arguments = action['name'], json.dumps(action['arguments'], ensure_ascii=False) - tool_calls.append(ToolCall(function=FunctionCall(name=name, arguments=arguments))) - - # get text outside of tags - if not text.startswith(''): - text = text[:text.find('')] - elif not text.endswith(''): - text = text[text.rfind('') + len(''):] - else: - text = '' - return ExtractedToolCallInformation(tools_called=True, - tool_calls=tool_calls, - content=text if len(text) > 0 else None) - - return ExtractedToolCallInformation(tools_called=False, tool_calls=[], content=text) + def parse_tool_call_complete(self, payload: str) -> ToolCall | None: + return self._parse_tool_call_complete_json(payload) diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py index 53c202b9f9..bb72ed1896 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py @@ -1,26 +1,13 @@ # Copyright (c) OpenMMLab. All rights reserved. -import json import re -from collections.abc import Sequence - -import partial_json_parser -import shortuuid -from partial_json_parser.core.options import Allow from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - DeltaFunctionCall, - DeltaMessage, DeltaToolCall, - ExtractedToolCallInformation, - FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -from .utils import find_common_prefix, is_complete_json logger = get_logger('lmdeploy') @@ -67,6 +54,13 @@ def get_tool_close_tag(self) -> str | None: def get_tool_payload_format(self) -> str: return 'json' + def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + """Decode Qwen3 JSON tool payload incrementally.""" + return self._decode_tool_incremental_json(added_text=added_text, final=final) + + def parse_tool_call_complete(self, payload: str) -> ToolCall | None: + return self._parse_tool_call_complete_json(payload) + def _split(self, parsing_content: str): """Split content into tuple: (text_content, tool_content, has_tool_end) @@ -94,136 +88,3 @@ def _split(self, parsing_content: str): parsing_content[start_idx + len(self.tool_start_token):end_idx], True, ) - - def detect_tool_start_tag( - self, - delta_text: str, - delta_token_ids: Sequence[int], - *, - stream_buffer: StreamBuffer, - request: ChatCompletionRequest, - ) -> int | None: - """Return index in delta_text where starts, if present. - - This is used by ResponseParser to split the chunk into reasoning vs tool-call portions without hard-coding - protocol details there. - """ - idx = delta_text.find(self.tool_start_token) - return idx if idx >= 0 else None - - def extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - """Extract tool calls from streaming model output.""" - current_text = stream_buffer.current_text - split_result = self._split(current_text[self.parse_cursor:]) - text_content, tool_content, has_tool_end = split_result - delta = DeltaMessage() - - if text_content: - delta.content = text_content - - if tool_content: - strip = tool_content.strip() - if strip: - flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR - obj: dict | None - try: - obj = partial_json_parser.loads(strip, flags) - except partial_json_parser.core.exceptions.MalformedJSON: - logger.debug('cannot parse into partial JSON yet') - obj = None - - if obj is not None and not self.current_tool_name_sent: - func_name = obj.get('name') - if func_name: - if not self.qwen_active_tool_call_id: - self.qwen_active_tool_call_id = f'chatcmpl-tool-{shortuuid.random()}' - self.qwen_tool_serial_index += 1 - self.streamed_args_for_tool.append('') - idx = self.qwen_tool_serial_index - delta.tool_calls = [ - DeltaToolCall( - id=self.qwen_active_tool_call_id, - index=idx, - type='function', - function=DeltaFunctionCall(name=func_name).model_dump(exclude_none=True), - ) - ] - self.current_tool_name_sent = True - self.prev_tool_call_arr = [dict(obj)] - elif obj is not None: - idx = self.qwen_tool_serial_index - args = self.get_argments(obj) - cur_arguments = args if isinstance(args, dict) else None - prev_arguments = ( - self.get_argments(self.prev_tool_call_arr[0]) if self.prev_tool_call_arr else None - ) - is_comp = is_complete_json(strip) - argument_diff = None - if cur_arguments: - cur_args_json = json.dumps(cur_arguments, ensure_ascii=False) - if is_comp: - sent = len(self.streamed_args_for_tool[idx]) - argument_diff = cur_args_json[sent:] - elif prev_arguments: - prev_args_json = json.dumps(prev_arguments, ensure_ascii=False) - if cur_args_json != prev_args_json: - prefix = find_common_prefix(prev_args_json, cur_args_json) - sent = len(self.streamed_args_for_tool[idx]) - argument_diff = prefix[sent:] - if argument_diff is not None: - delta.tool_calls = [ - DeltaToolCall( - index=idx, - id=self.qwen_active_tool_call_id, - function=DeltaFunctionCall( - arguments=argument_diff).model_dump(exclude_none=True), - ) - ] - self.streamed_args_for_tool[idx] += argument_diff - self.prev_tool_call_arr = [obj] - - if has_tool_end: - self.qwen_active_tool_call_id = '' - self.current_tool_name_sent = False - self.prev_tool_call_arr = [] - - return delta if delta.content is not None or delta.tool_calls else None - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - """Extract tool calls from complete model output. - - This method processes the full model output to extract tool calls, reasoning content, and regular text content. - Unlike the streaming version, this processes the entire output at once. - """ - text = model_output - - buf = [] - scan_pos = 0 - tool_calls = [] - for idx, match in enumerate(self.tool_call_pattern.finditer(text)): - buf.append(text[scan_pos:match.start()]) - scan_pos = match.end() - action = json.loads(match.group(1)) - name, arguments = action['name'], json.dumps(action['arguments'], ensure_ascii=False) - tool_calls.append(ToolCall(function=FunctionCall(name=name, arguments=arguments))) - if scan_pos < len(text): - buf.append(text[scan_pos:]) - text = ''.join(buf) - - return ExtractedToolCallInformation( - content=text, - tool_calls=tool_calls, - tools_called=bool(tool_calls), - ) diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index b458c8b292..a44498cd3b 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -1,21 +1,15 @@ # Copyright (c) OpenMMLab. All rights reserved. import json import re -from collections.abc import Sequence from typing import Any -import shortuuid - from lmdeploy.serve.openai.protocol import ( ChatCompletionRequest, DeltaFunctionCall, - DeltaMessage, DeltaToolCall, - ExtractedToolCallInformation, FunctionCall, ToolCall, ) -from lmdeploy.serve.openai.response_parser import StreamBuffer from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager @@ -113,6 +107,70 @@ def get_tool_close_tag(self) -> str | None: def get_tool_payload_format(self) -> str: return 'xml' + def start_tool_call(self) -> None: + super().start_tool_call() + self.coder_has_emitted_name = False + self.coder_has_emitted_json_start = False + self.coder_json_closed = False + self.coder_emitted_param_names.clear() + + def finish_tool_call(self) -> None: + super().finish_tool_call() + self.coder_has_emitted_name = False + self.coder_has_emitted_json_start = False + self.coder_json_closed = False + self.coder_emitted_param_names.clear() + + def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + """Decode XML tool payload incrementally into OpenAI tool-call + deltas.""" + self._tool_payload += added_text + func_name, args_dict, is_func_closed = self._extract_params(self._tool_payload) + + out: list[DeltaToolCall] = [] + if func_name and not self.coder_has_emitted_name: + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type='function', + function=DeltaFunctionCall(name=func_name), + )) + self.coder_has_emitted_name = True + + json_fragments: list[str] = [] + if not self.coder_has_emitted_json_start and (args_dict or is_func_closed): + json_fragments.append('{') + self.coder_has_emitted_json_start = True + + for k, v in args_dict.items(): + if k in self.coder_emitted_param_names: + continue + prefix = ', ' if len(self.coder_emitted_param_names) > 0 else '' + json_fragments.append(f'{prefix}\"{k}\": {json.dumps(v, ensure_ascii=False)}') + self.coder_emitted_param_names.add(k) + + if is_func_closed and self.coder_has_emitted_json_start and not self.coder_json_closed: + json_fragments.append('}') + self.coder_json_closed = True + + if json_fragments: + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=''.join(json_fragments)), + )) + return out + + def parse_tool_call_complete(self, payload: str) -> ToolCall | None: + func_name, args_dict, _ = self._extract_params(payload) + if not func_name: + return None + args_json = json.dumps(args_dict, ensure_ascii=False) if args_dict else '{}' + return ToolCall(function=FunctionCall(name=func_name, arguments=args_json)) + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: messages = request.messages if not isinstance(messages, list): @@ -191,130 +249,3 @@ def _extract_params(self, content: str) -> tuple[str | None, dict[str, Any], boo is_func_closed = self.func_end_token in content return func_name, args_dict, is_func_closed - - def detect_tool_start_tag( - self, - delta_text: str, - delta_token_ids: Sequence[int], - *, - stream_buffer: StreamBuffer, - request: ChatCompletionRequest, - ) -> int | None: - """Return index in ``delta_text`` where ```` starts.""" - text = stream_buffer.current_text - start_idx = text.rfind(self.tool_start_token) - end_idx = text.rfind(self.tool_end_token) - if start_idx >= 0 and end_idx < start_idx: - return 0 - idx = delta_text.find(self.tool_start_token) - return idx if idx >= 0 else None - - def extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - current_text = stream_buffer.current_text - - split_result = self._split(current_text[self.parse_cursor:]) - text_content, tool_content, has_tool_end = split_result - - delta = DeltaMessage() - if text_content: - delta.content = text_content - - if tool_content: - if not self.qwen_active_tool_call_id: - self.qwen_active_tool_call_id = f'chatcmpl-tool-{shortuuid.random()}' - self.qwen_tool_serial_index += 1 - self.coder_has_emitted_name = False - self.coder_has_emitted_json_start = False - self.coder_json_closed = False - self.coder_emitted_param_names.clear() - - func_name, args_dict, is_func_closed = self._extract_params(tool_content) - - fcall_delta = DeltaFunctionCall() - has_updates = False - - if func_name and not self.coder_has_emitted_name: - fcall_delta.name = func_name - self.coder_has_emitted_name = True - has_updates = True - - json_fragments = [] - if not self.coder_has_emitted_json_start: - if args_dict or is_func_closed: - json_fragments.append('{') - self.coder_has_emitted_json_start = True - - for k, v in args_dict.items(): - if k not in self.coder_emitted_param_names: - prefix = ', ' if len(self.coder_emitted_param_names) > 0 else '' - serialized = json.dumps(v, ensure_ascii=False) - json_fragments.append(f'{prefix}\"{k}\": {serialized}') - self.coder_emitted_param_names.add(k) - - if is_func_closed and not self.coder_json_closed: - if self.coder_has_emitted_json_start: - json_fragments.append('}') - self.coder_json_closed = True - - joined_fragments = ''.join(json_fragments) - if joined_fragments: - fcall_delta.arguments = joined_fragments - has_updates = True - - if has_updates: - parsed_delta = DeltaToolCall( - id=self.qwen_active_tool_call_id, - index=self.qwen_tool_serial_index, - function=fcall_delta, - ) - delta.tool_calls = [parsed_delta] - - if has_tool_end: - self.qwen_active_tool_call_id = '' - self.coder_has_emitted_name = False - self.coder_has_emitted_json_start = False - self.coder_json_closed = False - self.coder_emitted_param_names.clear() - - return delta - - def extract_tool_calls( - self, - model_output: str, - request: ChatCompletionRequest, - ) -> ExtractedToolCallInformation: - text = model_output - buf = [] - scan_pos = 0 - tool_calls = [] - - for idx, match in enumerate(self.tool_call_pat.finditer(text)): - buf.append(text[scan_pos:match.start()]) - scan_pos = match.end() - - tool_content = match.group(1) - func_name, args_dict, _ = self._extract_params(tool_content) - - if func_name: - tool_calls.append( - ToolCall(function=FunctionCall( - name=func_name, arguments=json.dumps(args_dict, ensure_ascii=False) if args_dict else '{}'))) - - if scan_pos < len(text): - buf.append(text[scan_pos:]) - - text = ''.join(buf) - - return ExtractedToolCallInformation( - content=text, - tool_calls=tool_calls, - tools_called=bool(tool_calls), - ) diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index 67b4bbcb7a..bafa91242a 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -1,12 +1,20 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/tool_parsers -from collections.abc import Sequence +import json from functools import cached_property +import partial_json_parser +import shortuuid from mmengine import Registry - -from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaMessage, ExtractedToolCallInformation -from lmdeploy.serve.openai.response_parser import StreamBuffer +from partial_json_parser.core.options import Allow + +from lmdeploy.serve.openai.protocol import ( + ChatCompletionRequest, + DeltaFunctionCall, + DeltaToolCall, + FunctionCall, + ToolCall, +) from lmdeploy.utils import get_logger logger = get_logger('lmdeploy') @@ -21,6 +29,15 @@ class ToolParser: def __init__(self, tokenizer: object): self.model_tokenizer = tokenizer + self._tool_payload: str = '' + self._active_tool_call_id: str = '' + self._active_tool_index: int = -1 + self._name_emitted: bool = False + self._args_prefix_emitted: bool = False + self._value_chars_emitted: int = 0 + self._args_closed_emitted: bool = False + self._args_emitted_len: int = 0 + self._prev_args_json: str | None = None @cached_property def vocab(self) -> dict[str, int]: @@ -40,58 +57,6 @@ def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionReques request.tools = [item.function.model_dump() for item in request.tools] return request - def extract_tool_calls(self, model_output: str, request: ChatCompletionRequest) -> ExtractedToolCallInformation: - """Static method that should be implemented for extracting tool calls - from a complete model-generated string. - - Used for non-streaming responses where we have the entire model response available before sending to the client. - Static because it's stateless. - """ - raise NotImplementedError('AbstractToolParser.extract_tool_calls has not been implemented!') - - def extract_tool_calls_streaming( - self, - delta_text: str, - delta_token_ids: Sequence[int], - request: ChatCompletionRequest, - *, - stream_buffer: StreamBuffer, - **kwargs, - ) -> DeltaMessage | None: - """Instance method that should be implemented for extracting tool calls - from an incomplete response; for use when handling tool calls and - streaming. - - Args: - delta_text: The new text chunk for this iteration. - delta_token_ids: The new token ids for this chunk. - request: The chat completion request. - stream_buffer: Cumulative decoding state (``ResponseParser`` or a test - double); use ``stream_buffer.current_text`` for the full partial output. - Tool-specific - fields live on the parser instance (one instance per request). - - Instance method because streaming uses the shared buffer plus parser-local state. - """ - raise NotImplementedError('AbstractToolParser.extract_tool_calls_streaming has not been ' - 'implemented!') - - def detect_tool_start_tag( - self, - delta_text: str, - delta_token_ids: Sequence[int], - *, - stream_buffer: StreamBuffer, - request: ChatCompletionRequest, - ) -> int | None: - """Optional hint for where tool-call protocol starts in *delta_text*. - - Default implementation returns None, meaning "no tool start detected in this chunk". Concrete parsers can - override this to let ResponseParser know where to split reasoning vs tool content without hard-coding any - protocol details here. - """ - return None - def get_tool_open_tag(self) -> str | None: """Return tool opening tag string, or None if unsupported.""" raise NotImplementedError('ToolParser.get_tool_open_tag has not been implemented!') @@ -103,3 +68,179 @@ def get_tool_close_tag(self) -> str | None: def get_tool_payload_format(self) -> str: """Return payload format for tool call body.""" raise NotImplementedError('ToolParser.get_tool_payload_format has not been implemented!') + + def start_tool_call(self) -> None: + """Mark start of a tool-call block.""" + self._active_tool_index += 1 + self._active_tool_call_id = f'chatcmpl-tool-{shortuuid.random()}' + self._name_emitted = False + self._args_prefix_emitted = False + self._value_chars_emitted = 0 + self._args_closed_emitted = False + self._args_emitted_len = 0 + self._prev_args_json = None + self._tool_payload = '' + + def finish_tool_call(self) -> None: + """Mark end of a tool-call block.""" + self._active_tool_call_id = '' + self._name_emitted = False + self._args_prefix_emitted = False + self._value_chars_emitted = 0 + self._args_closed_emitted = False + self._args_emitted_len = 0 + self._prev_args_json = None + self._tool_payload = '' + + def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + """Decode incremental tool payload emitted between tool tags.""" + raise NotImplementedError('ToolParser.decode_tool_incremental has not been implemented!') + + def parse_tool_call_complete(self, payload: str) -> ToolCall | None: + """Parse one complete tool payload into OpenAI tool call object.""" + raise NotImplementedError('ToolParser.parse_tool_call_complete has not been implemented!') + + def _decode_tool_incremental_json(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: + self._tool_payload += added_text + payload = self._tool_payload.strip() + if not payload: + return [] + + flags = Allow.ALL if self._name_emitted else Allow.ALL & ~Allow.STR + try: + obj = partial_json_parser.loads(payload, flags) + except partial_json_parser.core.exceptions.MalformedJSON: + return [] + if not isinstance(obj, dict): + return [] + + out: list[DeltaToolCall] = [] + if not self._name_emitted: + fn_name = obj.get('name') + if isinstance(fn_name, str) and fn_name: + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type='function', + function=DeltaFunctionCall(name=fn_name), + )) + self._name_emitted = True + + args_obj = obj.get('arguments', obj.get('parameters', None)) + if args_obj is None: + return out + + if isinstance(args_obj, dict): + items = list(args_obj.items()) + if not self._args_prefix_emitted and items: + first_key = items[0][0] + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=f'{{\"{first_key}\": \"')), + ) + self._args_prefix_emitted = True + + values_concat = ''.join(v for _, v in items if isinstance(v, str)) + if len(values_concat) > self._value_chars_emitted: + diff = values_concat[self._value_chars_emitted:] + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=diff), + )) + self._value_chars_emitted = len(values_concat) + + if self._is_complete_json(payload) and self._args_prefix_emitted and not self._args_closed_emitted: + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type=None, + function=DeltaFunctionCall(arguments='"}'), + )) + self._args_closed_emitted = True + return out + + args_json = json.dumps(args_obj, ensure_ascii=False) + if args_json in ('{}', '[]'): + return out + + emitted_arg = False + candidate: str | None = None + if self._is_complete_json(payload): + candidate = args_json + elif self._prev_args_json: + candidate = self._common_prefix(self._prev_args_json, args_json) + elif self._args_emitted_len == 0 and added_text: + pos = args_json.find(added_text) + if pos >= 0: + candidate = args_json[:pos + len(added_text)] + + if candidate and len(candidate) > self._args_emitted_len: + diff = candidate[self._args_emitted_len:] + if final or any(ch.isalnum() for ch in diff): + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=diff), + )) + self._args_emitted_len = len(candidate) + emitted_arg = True + + if ( + not emitted_arg + and self._args_emitted_len > 0 + and added_text + and any(ord(ch) > 127 for ch in added_text) + ): + out.append( + DeltaToolCall( + id=self._active_tool_call_id, + index=self._active_tool_index, + type=None, + function=DeltaFunctionCall(arguments=added_text), + )) + self._args_emitted_len += len(added_text) + self._prev_args_json = args_json + return out + + @staticmethod + def _is_complete_json(text: str) -> bool: + try: + json.loads(text) + return True + except json.JSONDecodeError: + return False + + @staticmethod + def _common_prefix(s1: str, s2: str) -> str: + i = 0 + n = min(len(s1), len(s2)) + while i < n and s1[i] == s2[i]: + i += 1 + return s1[:i] + + @staticmethod + def _parse_tool_call_complete_json(payload: str) -> ToolCall | None: + if not payload: + return None + try: + obj = json.loads(payload) + except json.JSONDecodeError: + return None + if not isinstance(obj, dict): + return None + name = obj.get('name') + if not isinstance(name, str) or not name: + return None + args_obj = obj.get('arguments', obj.get('parameters', {})) + args_json = json.dumps(args_obj, ensure_ascii=False) + return ToolCall(function=FunctionCall(name=name, arguments=args_json)) diff --git a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py index 769c927e34..3d9246c6c9 100644 --- a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py +++ b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py @@ -6,7 +6,7 @@ from lmdeploy.serve.openai.tool_parser.qwen3_tool_parser import Qwen3ToolParser from lmdeploy.tokenizer import HuggingFaceTokenizer -MODEL_ID = 'Qwen/Qwen3-8B' +MODEL_ID = '/nvme4/huggingface_hub/hub/models--Qwen--Qwen3-8B/snapshots/1808139acb3a01b52eb3a2cf54defbc8a163146e' @pytest.fixture(scope='module') @@ -233,17 +233,161 @@ def _call(delta_text: str): assert delta_msg.content is None assert tool_emitted is False - # 3) chunk carries reasoning end + normal content + # 3) chunk carries reasoning end + normal content. + # New parser emits ordered events, so this call emits reasoning first. delta_msg, tool_emitted = _call('The answer is 9 OK. The') assert delta_msg is not None assert delta_msg.reasoning_content == 'The answer is 9 ' - assert delta_msg.content == ' OK. The' + assert delta_msg.content is None assert tool_emitted is False - # 4) chunk carries stray think-close + content + tool-open + # Next call flushes queued plain content from previous chunk first. delta_msg, tool_emitted = _call('fine. \n\n ') assert delta_msg is not None + assert delta_msg.reasoning_content is None + assert delta_msg.content == ' OK. The' + assert tool_emitted is False + + # Flush the next queued plain segment from chunk-4. + delta_msg, tool_emitted = _call('') + assert delta_msg is not None # Stray closing tag after reasoning has ended is treated as plain content. assert delta_msg.reasoning_content is None assert delta_msg.content == 'fine. \n\n ' assert tool_emitted is False + + def test_stream_chunk_tool_enabled_without_reasoning_parser(self, tokenizer): + """When reasoning parser is disabled, tool parsing still works. + + This proves the tool branch is reachable from plain mode after seeing the tool open tag, even with no reasoning + parser configured. + """ + old_reasoning_cls = ResponseParser.reasoning_parser_cls + old_tool_cls = ResponseParser.tool_parser_cls + try: + ResponseParser.reasoning_parser_cls = None + ResponseParser.tool_parser_cls = Qwen3ToolParser + + request = ChatCompletionRequest( + model=MODEL_ID, + messages=[], + stream=True, + tool_choice='auto', + chat_template_kwargs={'enable_thinking': False}, + ) + parser = ResponseParser(request=request, tokenizer=tokenizer) + + chunks = [ + 'prefix ', + '', + '\n', + '{"', + 'name', + '":', + ' "', + 'get', + '_weather', + '",', + ] + tool_seen = False + for chunk in chunks: + delta_ids = self._encode_ids(tokenizer, chunk) + delta_msg, tool_emitted = parser.stream_chunk(delta_text=chunk, delta_token_ids=delta_ids) + if delta_msg is not None: + assert delta_msg.reasoning_content is None + if tool_emitted: + tool_seen = True + assert delta_msg is not None + assert delta_msg.tool_calls is not None + assert delta_msg.tool_calls[0].function is not None + assert delta_msg.tool_calls[0].function.name == 'get_weather' + assert tool_seen is True + finally: + ResponseParser.reasoning_parser_cls = old_reasoning_cls + ResponseParser.tool_parser_cls = old_tool_cls + + def test_stream_chunk_reasoning_without_open_tag(self, tokenizer, response_parser): + """Qwen thinking mode may omit ```` and start directly with + reasoning. + + In this case, chunks before ```` must be emitted as + ``reasoning_content``. + """ + + def _call(delta_text: str): + delta_ids = self._encode_ids(tokenizer, delta_text) + return response_parser.stream_chunk(delta_text=delta_text, delta_token_ids=delta_ids) + + # No opening tag, but still in reasoning mode initially. + delta_msg, tool_emitted = _call('Let me reason ') + assert delta_msg is not None + assert delta_msg.reasoning_content == 'Let me reason ' + assert delta_msg.content is None + assert tool_emitted is False + + delta_msg, tool_emitted = _call('step by step') + assert delta_msg is not None + assert delta_msg.reasoning_content == 'step by step' + assert delta_msg.content is None + assert tool_emitted is False + + # Closing tag chunk itself is swallowed. + delta_msg, tool_emitted = _call('') + assert delta_msg is None + assert tool_emitted is False + + # After close tag, emit normal content. + delta_msg, tool_emitted = _call(' final answer') + assert delta_msg is not None + assert delta_msg.reasoning_content is None + assert delta_msg.content == ' final answer' + assert tool_emitted is False + + def test_stream_chunk_preserves_content_reasoning_content_order(self, tokenizer, response_parser): + """Mixed single chunk should preserve event order without content + merge.""" + class PlainStartQwenReasoningParser(QwenReasoningParser): + + def starts_in_reasoning_mode(self) -> bool: + return False + + old_reasoning_cls = ResponseParser.reasoning_parser_cls + old_tool_cls = ResponseParser.tool_parser_cls + try: + ResponseParser.reasoning_parser_cls = PlainStartQwenReasoningParser + ResponseParser.tool_parser_cls = Qwen3ToolParser + request = ChatCompletionRequest( + model=MODEL_ID, + messages=[], + stream=True, + tool_choice='auto', + chat_template_kwargs={'enable_thinking': True}, + ) + parser = ResponseParser(request=request, tokenizer=tokenizer) + + delta_text = 'content-xxx reasoning-yyy content-zzz ' + delta_ids = self._encode_ids(tokenizer, delta_text) + + # 1st event: plain content before + delta_msg, tool_emitted = parser.stream_chunk(delta_text=delta_text, delta_token_ids=delta_ids) + assert delta_msg is not None + assert delta_msg.content == 'content-xxx ' + assert delta_msg.reasoning_content is None + assert tool_emitted is False + + # 2nd event: reasoning segment + delta_msg, tool_emitted = parser.stream_chunk(delta_text='', delta_token_ids=[]) + assert delta_msg is not None + assert delta_msg.content is None + assert delta_msg.reasoning_content == ' reasoning-yyy ' + assert tool_emitted is False + + # 3rd event: trailing content segment before + delta_msg, tool_emitted = parser.stream_chunk(delta_text='', delta_token_ids=[]) + assert delta_msg is not None + assert delta_msg.content == ' content-zzz ' + assert delta_msg.reasoning_content is None + assert tool_emitted is False + finally: + ResponseParser.reasoning_parser_cls = old_reasoning_cls + ResponseParser.tool_parser_cls = old_tool_cls diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py deleted file mode 100644 index dda4d35806..0000000000 --- a/tests/test_lmdeploy/server/reasoning_parsers/test_deepseek_reasoning_parser.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. - -from __future__ import annotations - -import pytest -import transformers -from packaging.version import Version - -from lmdeploy.serve.openai.protocol import ChatCompletionRequest -from lmdeploy.serve.openai.reasoning_parser.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser -from lmdeploy.serve.openai.response_parser import StreamBuffer -from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer - -TRANSFORMERS_LT_5 = Version(transformers.__version__) < Version('5.0.0') -REQUIRES_TRANSFORMERS_LT_5 = pytest.mark.skipif( - not TRANSFORMERS_LT_5, - reason=f'requires transformers < 5.0, got {transformers.__version__}', -) -pytestmark = REQUIRES_TRANSFORMERS_LT_5 - - -MODEL_ID = 'deepseek-ai/DeepSeek-V3.1' - -@pytest.fixture(scope='module') -def tokenizer(): - try: - return HuggingFaceTokenizer(MODEL_ID) - except Exception as exc: # noqa: BLE001 - pytest.skip(f'Could not load tokenizer for {MODEL_ID}: {exc}') - - -def _make_request(stream: bool = False) -> ChatCompletionRequest: - return ChatCompletionRequest(model=MODEL_ID, messages=[], stream=stream) - - -def _build_parser(tokenizer: HuggingFaceTokenizer, *, enable_thinking: bool | None) -> DeepSeekV3ReasoningParser: - return DeepSeekV3ReasoningParser(tokenizer, enable_thinking=enable_thinking) - - -def simulate_pipeline_chunks( - tokenizer: HuggingFaceTokenizer, - full_text: str, - *, - chunk_size: int = 1, - skip_special_tokens: bool = True, - spaces_between_special_tokens: bool = True, -) -> list[tuple[str, list[int]]]: - all_ids = tokenizer.encode(full_text, add_bos=False, add_special_tokens=False) - state = DetokenizeState(0) - accumulated: list[int] = [] - chunks: list[tuple[str, list[int]]] = [] - offset = 0 - while offset < len(all_ids): - accumulated.extend(all_ids[offset:offset + chunk_size]) - offset += chunk_size - ids_offset_before = state.ids_offset - delta_text, state = tokenizer.detokenize_incrementally( - accumulated, - state, - skip_special_tokens=skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens, - ) - delta_ids = accumulated[ids_offset_before:len(accumulated)] - chunks.append((delta_text, delta_ids)) - return chunks - - -def run_reasoning_stream( - parser: DeepSeekV3ReasoningParser, - request: object, - chunks: list[tuple[str, list[int]]], -) -> tuple[str, str]: - state = StreamBuffer() - reasoning_acc = '' - content_acc = '' - for delta_text, delta_ids in chunks: - state.update(delta_text, delta_ids) - delta_msg = parser.extract_reasoning_streaming( - delta_text=delta_text or '', - delta_token_ids=delta_ids, - request=request, - stream_buffer=state, - ) - if delta_msg is not None: - if delta_msg.reasoning_content: - reasoning_acc += delta_msg.reasoning_content - if delta_msg.content is not None: - content_acc += delta_msg.content - state.step() - return reasoning_acc, content_acc - - -class TestExtractReasoning: - - def test_enable_thinking_true(self, tokenizer): - parser = _build_parser(tokenizer, enable_thinking=True) - full = '\nBrief chain of thought.\n\n\nThe answer is 42.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning == '\nBrief chain of thought.\n' - assert content == '\n\nThe answer is 42.' - - def test_enable_thinking_none(self, tokenizer): - parser = _build_parser(tokenizer, enable_thinking=None) - full = 'The answer is 42.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning is None - assert content == full - - -class TestExtractReasoningStreaming: - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_enable_thinking_true(self, tokenizer, chunk_size): - parser = _build_parser(tokenizer, enable_thinking=True) - full = '\nBrief chain of thought.\n\n\nThe answer is 42.' - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - r_stream, c_stream = run_reasoning_stream(parser, _make_request(stream=True), chunks) - r_ns, c_ns = parser.extract_reasoning(full, _make_request()) - assert r_stream == r_ns - assert c_stream == c_ns - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_enable_thinking_none(self, tokenizer, chunk_size): - parser = _build_parser(tokenizer, enable_thinking=False) - full = 'The answer is 42.' - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - r_stream, c_stream = run_reasoning_stream(parser, _make_request(stream=True), chunks) - assert r_stream == '' - assert c_stream == full diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_harmony_gpt_oss_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_harmony_gpt_oss_parser.py deleted file mode 100644 index 7624ff4d17..0000000000 --- a/tests/test_lmdeploy/server/reasoning_parsers/test_harmony_gpt_oss_parser.py +++ /dev/null @@ -1,328 +0,0 @@ -import collections -import json -import os -import sys -import time -import types -from collections.abc import Generator - -import pytest -import shortuuid - -# Ensure local package is imported (not any site-packages installation) -REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) -if REPO_ROOT not in sys.path: - sys.path.insert(0, REPO_ROOT) - - -def _install_openai_harmony_stub(): - """Install a minimal stub for `openai_harmony` so the module imports - without the real dependency. - - The GptOssChatParser test injects its own dummy parser, so the stub is sufficient. - """ - if 'openai_harmony' in sys.modules: - return - m = types.ModuleType('openai_harmony') - - class HarmonyEncodingName: - HARMONY_GPT_OSS = 'HARMONY_GPT_OSS' - - class Role: - ASSISTANT = 'assistant' - - class StreamableParser: # pragma: no cover - constructor only used - - def __init__(self, encoding, role=None): - self.encoding = encoding - self.role = role - - def load_harmony_encoding(name): # pragma: no cover - not used in test - return object() - - m.HarmonyEncodingName = HarmonyEncodingName - m.Role = Role - m.StreamableParser = StreamableParser - m.load_harmony_encoding = load_harmony_encoding - sys.modules['openai_harmony'] = m - - -TestExpects = collections.namedtuple('TestExpects', 'func_name location') - - -class DummyParser: - """A minimal stand-in for Harmony's StreamableParser with channels. - - Control tokens: - -1: start functions.get_weather (commentary) - -4: start functions.get_time (commentary) - -6: start functions.get_weather (again) - -9: end current tool call, append to `messages` - -2: switch to final (visible) content - -3: switch to analysis (reasoning) - Other tokens are interpreted as chr(token). - """ - - class _Msg: - - def __init__(self, channel, recipient): - self.channel = channel - self.recipient = recipient - - def __init__(self): - self.current_channel = None - self.current_recipient = None - self.last_content_delta = '' - self.messages = [] - - def process(self, token): - if token == -1: - self.current_channel = 'commentary' - self.current_recipient = 'functions.get_weather' - self.last_content_delta = '' - return - if token == -4: - self.current_channel = 'commentary' - self.current_recipient = 'functions.get_time' - self.last_content_delta = '' - return - if token == -6: - self.current_channel = 'commentary' - self.current_recipient = 'functions.get_weather' - self.last_content_delta = '' - return - if token == -9: - if self.current_channel == 'commentary' and self.current_recipient and self.current_recipient.startswith( - 'functions.'): - self.messages.append(self._Msg(self.current_channel, self.current_recipient)) - # reset recipient to signal end of current tool call - self.current_recipient = None - self.current_channel = None - self.last_content_delta = '' - return - if token == -2: - self.current_channel = 'final' - self.current_recipient = None - self.last_content_delta = '' - return - if token == -3: - self.current_channel = 'analysis' - self.current_recipient = None - self.last_content_delta = '' - return - # regular character token - self.last_content_delta = chr(token) - - -def _chat_completion_v1(request, token_chunks: list[list[int]]): - from lmdeploy.serve.openai.harmony_utils import GptOssChatParser - from lmdeploy.serve.openai.protocol import ( - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, - UsageInfo, - ) - - request_id = f'chat-{shortuuid.random()}' - created_time = int(time.time()) - model_name = request.model - - parser = GptOssChatParser() - parser.parser = DummyParser() - - if request.stream: - - def completion_stream_generator() -> Generator['ChatCompletionStreamResponse', None, None]: - finish_reason = 'stop' - for chunk in token_chunks: - delta_message = parser.parse_streaming(chunk) - choice_data = ChatCompletionResponseStreamChoice(index=0, - delta=delta_message, - finish_reason=finish_reason, - logprobs=None) - response = ChatCompletionStreamResponse(id=request_id, - created=created_time, - model=model_name, - choices=[choice_data], - usage=None) - yield response - - return completion_stream_generator() - - # Non-stream path: parse all tokens at once using parse_full - tokens: list[int] = [] - for c in token_chunks: - tokens.extend(c) - message = parser.parse_full(tokens) - finish_reason = 'tool_calls' if message.tool_calls else 'stop' - choice_data = ChatCompletionResponseChoice(index=0, message=message, finish_reason=finish_reason) - return ChatCompletionResponse(id=request_id, - created=created_time, - model=model_name, - choices=[choice_data], - usage=UsageInfo()) - - -def _stream_parse(request, token_chunks: list[list[int]]): - from lmdeploy.serve.openai.protocol import DeltaMessage - - content = '' - reasoning_content = '' - tool_calls_by_index = {} - - for i, stream_resp in enumerate(_chat_completion_v1(request, token_chunks)): - delta_message: DeltaMessage = stream_resp.choices[0].delta - if delta_message.content: - content += delta_message.content - if delta_message.reasoning_content: - reasoning_content += delta_message.reasoning_content - if delta_message.tool_calls: - for c in delta_message.tool_calls: - idx = c.index - existing_call = tool_calls_by_index.get(idx, None) - if not existing_call: - tool_calls_by_index[idx] = c - continue - if c.function.name: - existing_call.function.name = c.function.name - if c.function.arguments: - existing_call.function.arguments = existing_call.function.arguments or '' - existing_call.function.arguments += c.function.arguments - # sorted list for stable order - tool_calls = [tool_calls_by_index[i] for i in sorted(tool_calls_by_index.keys())] - return content, reasoning_content, tool_calls - - -def _t(s: str) -> list[int]: - return [ord(c) for c in s] - - -# Basic: single function call split across two chunks (bug repro scenario) -TOKENS_SINGLE_CALL_TWO_CHUNKS = [ - [-1] + _t('{"location": "Paris'), - _t(', France"}'), -] - -# Multiple calls with indices and different function names -TOKENS_TWO_CALLS_DIFFERENT_FUNCS = [ - [-1] + _t('{"location": "Berlin"}') + [-9] + [-4] + _t('{"city": "New'), - _t(' York"}') + [-9], -] - -# Interleaved channels: analysis, tool call, final content -TOKENS_INTERLEAVED = [ - [-3] + _t('Thinking about the weather. ') + [-1] + _t('{"location": "Par'), - _t('is, France"}') + [-9] + [-2] + _t('Fetching the weather now.'), -] - -# Two calls, same function name, indices increment -TOKENS_TWO_CALLS_SAME_FUNC = [ - [-1] + _t('{"location": "Tokyo"}') + [-9], - [-6] + _t('{"location": "Ky'), - _t('oto"}') + [-9], -] - - -@pytest.mark.parametrize(('token_chunks', 'expects'), [ - (TOKENS_SINGLE_CALL_TWO_CHUNKS, [TestExpects('get_weather', 'Paris, France')]), -]) -def test_parser_stream_basic(token_chunks: list[list[int]], expects: list[TestExpects]): - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - - _install_openai_harmony_stub() - request = ChatCompletionRequest(model='gpt-oss', messages=[], stream=True) - content, reasoning_content, tool_calls = _stream_parse(request, token_chunks) - - assert len(tool_calls) == len(expects) - for parsed_call, expected_call in zip(tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args['location'] == expected_call.location - assert content.strip() == '' - assert (reasoning_content or '').strip() == '' - - -def test_parser_stream_multiple_calls_indices(): - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - - _install_openai_harmony_stub() - request = ChatCompletionRequest(model='gpt-oss', messages=[], stream=True) - content, reasoning_content, tool_calls = _stream_parse(request, TOKENS_TWO_CALLS_DIFFERENT_FUNCS) - - assert len(tool_calls) == 2 - # tool_calls sorted by index ensures stable order - tc0, tc1 = tool_calls - assert tc0.index == 0 and tc1.index == 1 - assert tc0.function.name == 'get_weather' - assert json.loads(tc0.function.arguments)['location'] == 'Berlin' - assert tc1.function.name == 'get_time' - assert json.loads(tc1.function.arguments)['city'] == 'New York' - assert (content or '').strip() == '' - assert (reasoning_content or '').strip() == '' - - -def test_parser_stream_interleaved_channels(): - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - - _install_openai_harmony_stub() - request = ChatCompletionRequest(model='gpt-oss', messages=[], stream=True) - content, reasoning_content, tool_calls = _stream_parse(request, TOKENS_INTERLEAVED) - - assert json.loads(tool_calls[0].function.arguments)['location'] == 'Paris, France' - assert reasoning_content == 'Thinking about the weather. ' - assert content == 'Fetching the weather now.' - - -@pytest.mark.parametrize(('token_chunks', 'expects'), [ - (TOKENS_TWO_CALLS_SAME_FUNC, [TestExpects('get_weather', 'Tokyo'), - TestExpects('get_weather', 'Kyoto')]), -]) -def test_parser_stream_two_calls_same_func(token_chunks: list[list[int]], expects: list[TestExpects]): - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - - _install_openai_harmony_stub() - request = ChatCompletionRequest(model='gpt-oss', messages=[], stream=True) - _, _, tool_calls = _stream_parse(request, token_chunks) - - assert len(tool_calls) == len(expects) - for parsed_call, expected_call in zip(tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args['location'] == expected_call.location - - -def test_open_tool_call_no_args(): - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - - _install_openai_harmony_stub() - request = ChatCompletionRequest(model='gpt-oss', messages=[], stream=True) - content, reasoning_content, tool_calls = _stream_parse(request, [[-1]]) - - assert len(tool_calls) == 1 - assert tool_calls[0].function.name == 'get_weather' - assert (tool_calls[0].function.arguments or '') == '' - assert (content or '') == '' - assert (reasoning_content or '') == '' - - -@pytest.mark.parametrize(('token_chunks', 'expects'), [ - (TOKENS_SINGLE_CALL_TWO_CHUNKS, [TestExpects('get_weather', 'Paris, France')]), - (TOKENS_TWO_CALLS_SAME_FUNC, [TestExpects('get_weather', 'Tokyo'), - TestExpects('get_weather', 'Kyoto')]), -]) -def test_parser_nonstream(token_chunks: list[list[int]], expects: list[TestExpects]): - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - - _install_openai_harmony_stub() - resp = _chat_completion_v1(ChatCompletionRequest(model='gpt-oss', messages=[], stream=False), token_chunks) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert first_message.content is None - assert (first_message.reasoning_content or '') == '' - assert len(first_message.tool_calls) == len(expects) - for parsed_call, expected_call in zip(first_message.tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args['location'] == expected_call.location diff --git a/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py b/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py deleted file mode 100644 index d576db4ce3..0000000000 --- a/tests/test_lmdeploy/server/reasoning_parsers/test_qwen_reasoning_parser.py +++ /dev/null @@ -1,266 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -"""Tests for QwenReasoningParser covering three model behavior modes. - -Scenario A – Thinking mode (Qwen3-8B, enable_thinking=True): - Model generates ``reasoning\\n\\nAnswer``. - -Scenario B – Non-thinking mode (Qwen3-8B, enable_thinking=False): - Model generates plain content with no ```` tags at all. - -Scenario C – Forceful Thinking (Qwen3-4B-Thinking-2507): - ```` is injected into the prompt by the chat template, so the - model's output starts directly with reasoning, then ````, then - the answer. No ```` appears in the generated output. -""" - -from __future__ import annotations - -import pytest - -from lmdeploy.serve.openai.protocol import ChatCompletionRequest -from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager -from lmdeploy.serve.openai.response_parser import StreamBuffer -from lmdeploy.tokenizer import DetokenizeState, HuggingFaceTokenizer - -# We use Qwen3-8B's tokenizer to simulate all the test cases. -MODEL_ID = 'Qwen/Qwen3-8B' - -@pytest.fixture(scope='module') -def tokenizer(): - try: - return HuggingFaceTokenizer(MODEL_ID) - except Exception as exc: # noqa: BLE001 - pytest.skip(f'Could not load tokenizer for {MODEL_ID}: {exc}') - - -@pytest.fixture() -def parser(tokenizer): - return QwenReasoningParser(tokenizer) - - -def simulate_pipeline_chunks( - tokenizer: HuggingFaceTokenizer, - full_text: str, - *, - chunk_size: int = 1, - skip_special_tokens: bool = True, - spaces_between_special_tokens: bool = True, -) -> list[tuple[str, list[int]]]: - """Split *full_text* into (delta_text, delta_token_ids) like - ``AsyncEngine.generate``.""" - all_ids = tokenizer.encode(full_text, add_bos=False, add_special_tokens=False) - state = DetokenizeState(0) - accumulated: list[int] = [] - chunks: list[tuple[str, list[int]]] = [] - offset = 0 - while offset < len(all_ids): - accumulated.extend(all_ids[offset:offset + chunk_size]) - offset += chunk_size - ids_offset_before = state.ids_offset - delta_text, state = tokenizer.detokenize_incrementally( - accumulated, - state, - skip_special_tokens=skip_special_tokens, - spaces_between_special_tokens=spaces_between_special_tokens, - ) - delta_ids = accumulated[ids_offset_before:len(accumulated)] - chunks.append((delta_text, delta_ids)) - return chunks - - -def run_reasoning_stream( - parser: QwenReasoningParser, - request: object, - chunks: list[tuple[str, list[int]]], -) -> tuple[str, str]: - """Mirror ``api_server`` ``completion_stream_generator`` parser loop. - - Returns (accumulated_reasoning, accumulated_content). - """ - state = StreamBuffer() - reasoning_acc = '' - content_acc = '' - for delta_text, delta_ids in chunks: - state.update(delta_text, delta_ids) - delta_msg = parser.extract_reasoning_streaming( - delta_text=delta_text or '', - delta_token_ids=delta_ids, - request=request, - stream_buffer=state, - ) - if delta_msg is not None: - if delta_msg.reasoning_content: - reasoning_acc += delta_msg.reasoning_content - if delta_msg.content is not None: - content_acc += delta_msg.content - state.step() - return reasoning_acc, content_acc - - -def _make_request(stream: bool = False) -> ChatCompletionRequest: - return ChatCompletionRequest(model=MODEL_ID, messages=[], stream=stream) - - -class TestExtractReasoning: - """Non-streaming ``extract_reasoning`` tests.""" - - def test_thinking_mode(self, parser): - """Qwen3-8B enable_thinking=True: - - ..reasoning..answer. - """ - full = '\nBrief chain of thought.\n\n\nThe answer is 42.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning == '\nBrief chain of thought.\n' - assert content == '\n\nThe answer is 42.' - - def test_non_thinking_mode(self, parser): - """Qwen3-8B enable_thinking=False: plain content, no tags.""" - full = 'The answer is 42.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning is None - assert content == 'The answer is 42.' - - def test_forceful_thinking(self, parser): - """Qwen3-4B-Thinking-2507: no in output, model starts with reasoning.""" - full = '\nBrief chain of thought.\n\n\nThe answer is 42.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning == '\nBrief chain of thought.\n' - assert content == '\n\nThe answer is 42.' - - def test_empty_reasoning(self, parser): - """Edge case: with empty reasoning body.""" - full = '\n\nThe answer is 42.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning is None - assert content == '\n\nThe answer is 42.' - - def test_only_reasoning_no_answer(self, parser): - """Edge case: reasoning present but no content after .""" - full = 'reasoning only' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning == 'reasoning only' - assert content is None - - def test_multiline_reasoning(self, parser): - """Longer, multi-line reasoning body.""" - reasoning_text = ( - '\nStep 1: identify the problem.\n' - 'Step 2: solve it.\n' - 'Step 3: verify.\n' - ) - full = f'{reasoning_text}\n\nFinal answer.' - reasoning, content = parser.extract_reasoning(full, _make_request()) - assert reasoning == reasoning_text - assert content == '\n\nFinal answer.' - - -class TestExtractReasoningStreaming: - """Streaming ``extract_reasoning_streaming`` tests. - - Each test is parametrized over chunk_size to exercise both fine-grained (token-by-token) and coarse (multi-token) - chunk boundaries. - """ - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_thinking_mode(self, tokenizer, parser, chunk_size): - """Qwen3-8B enable_thinking=True: streaming output matches non- - streaming.""" - reasoning_body = '\nBrief chain of thought.\n' - answer = 'The answer is 42.' - full = f'{reasoning_body}\n\n{answer}' - - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - request = _make_request(stream=True) - r_stream, c_stream = run_reasoning_stream(parser, request, chunks) - - r_ns, c_ns = parser.extract_reasoning(full, _make_request()) - assert r_stream == r_ns - assert c_stream == c_ns - assert answer in c_stream - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_forceful_thinking(self, tokenizer, parser, chunk_size): - """Qwen3-4B-Thinking-2507: no , streaming matches non-streaming.""" - reasoning_body = '\nBrief chain of thought.\n' - answer = 'The answer is 42.' - full = f'{reasoning_body}\n\n{answer}' - - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - request = _make_request(stream=True) - r_stream, c_stream = run_reasoning_stream(parser, request, chunks) - - r_ns, c_ns = parser.extract_reasoning(full, _make_request()) - assert r_stream == r_ns - assert c_stream == c_ns - assert answer in c_stream - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_non_thinking_mode(self, tokenizer, parser, chunk_size): - """Qwen3-8B enable_thinking=False: no tags at all. - - The streaming parser has no way to know that will never arrive, so it treats all text as - reasoning_content. The non-streaming path correctly returns it as content because it can inspect the full - output. This test documents the streaming behavior. - """ - full = 'The answer is 42.' - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - request = _make_request(stream=True) - r_stream, c_stream = run_reasoning_stream(parser, request, chunks) - - assert r_stream == full - assert c_stream == '' - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_empty_reasoning(self, tokenizer, parser, chunk_size): - """Edge case: with empty reasoning body.""" - answer = 'The answer is 42.' - full = f'\n\n{answer}' - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - request = _make_request(stream=True) - r_stream, c_stream = run_reasoning_stream(parser, request, chunks) - - assert r_stream == '' - assert answer in c_stream - - @pytest.mark.parametrize('chunk_size', [1, 3]) - def test_multiline_reasoning(self, tokenizer, parser, chunk_size): - """Longer reasoning body, streaming matches non-streaming.""" - reasoning_text = ( - '\nStep 1: identify the problem.\n' - 'Step 2: solve it.\n' - 'Step 3: verify.\n' - ) - answer = 'Final answer.' - full = f'{reasoning_text}\n\n{answer}' - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=chunk_size) - request = _make_request(stream=True) - r_stream, c_stream = run_reasoning_stream(parser, request, chunks) - - r_ns, c_ns = parser.extract_reasoning(full, _make_request()) - assert r_stream == r_ns - assert c_stream == c_ns - assert answer in c_stream - - -class TestRegistry: - - @pytest.mark.parametrize('name', ['qwen-qwq', 'qwen3', 'intern-s1', 'deepseeek-r1']) - def test_registered_names(self, tokenizer, name): - """All registered aliases resolve to QwenReasoningParser.""" - cls = ReasoningParserManager.get(name) - parser = cls(tokenizer) - assert isinstance(parser, QwenReasoningParser) - - def test_basic_stream_round_trip(self, tokenizer): - """Sanity check: registry-created parser works end-to-end.""" - cls = ReasoningParserManager.get('qwen3') - parser = cls(tokenizer) - full = f'{QwenReasoningParser.start_token}x{QwenReasoningParser.end_token}y' - chunks = simulate_pipeline_chunks(tokenizer, full, chunk_size=2) - request = _make_request(stream=True) - r_stream, c_stream = run_reasoning_stream(parser, request, chunks) - r_ns, c_ns = parser.extract_reasoning(full, _make_request()) - assert r_stream == r_ns - assert c_stream == c_ns diff --git a/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py b/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py deleted file mode 100644 index 3159181af4..0000000000 --- a/tests/test_lmdeploy/server/tool_parsers/test_qwen3_parser.py +++ /dev/null @@ -1,441 +0,0 @@ -import json -import time -from collections.abc import Generator - -import pytest -import shortuuid - -from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, - ChatMessage, - DeltaMessage, - UsageInfo, -) -from lmdeploy.serve.openai.reasoning_parser import QwenReasoningParser -from lmdeploy.serve.openai.response_parser import StreamBuffer -from lmdeploy.serve.openai.tool_parser import Qwen3ToolParser -from lmdeploy.tokenizer import Tokenizer - - -@pytest.fixture(scope='module') -def tokenizer(): - from lmdeploy.tokenizer import HuggingFaceTokenizer - return HuggingFaceTokenizer('Qwen/Qwen3-8B') - -@pytest.fixture() -def reasoning_parser(tokenizer): - return QwenReasoningParser(tokenizer) - -@pytest.fixture() -def tool_parser(tokenizer): - return Qwen3ToolParser(tokenizer) - -DELTA_TEXT_SEQUENCE = [ - # (delta_text, reasoning_content, content, tool_calls) - ('', None, None, []), - ('\n', '\n', None, []), - ('好的', '好的', None, []), - ('，', '，', None, []), - ('用户', '用户', None, []), - ('问', '问', None, []), - ('的是', '的是', None, []), - ('北京', '北京', None, []), - ('的', '的', None, []), - ('天气', '天气', None, []), - ('怎么样', '怎么样', None, []), - ('。', '。', None, []), - ('我', '我', None, []), - ('需要', '需要', None, []), - ('调', '调', None, []), - ('用', '用', None, []), - ('get', 'get', None, []), - ('_weather', '_weather', None, []), - ('这个', '这个', None, []), - ('工具', '工具', None, []), - ('来', '来', None, []), - ('获取', '获取', None, []), - ('信息', '信息', None, []), - ('。', '。', None, []), - ('首先', '首先', None, []), - ('，', '，', None, []), - ('确认', '确认', None, []), - ('用户', '用户', None, []), - ('提供的', '提供的', None, []), - ('地点', '地点', None, []), - ('是', '是', None, []), - ('北京', '北京', None, []), - ('，', '，', None, []), - ('参数', '参数', None, []), - ('正确', '正确', None, []), - ('。', '。', None, []), - ('然后', '然后', None, []), - ('检查', '检查', None, []), - ('工具', '工具', None, []), - ('的', '的', None, []), - ('参数', '参数', None, []), - ('要求', '要求', None, []), - ('，', '，', None, []), - ('只需要', '只需要', None, []), - ('location', 'location', None, []), - ('，', '，', None, []), - ('类型', '类型', None, []), - ('是', '是', None, []), - ('字符串', '字符串', None, []), - ('。', '。', None, []), - ('于是', '于是', None, []), - ('构造', '构造', None, []), - ('参数', '参数', None, []), - ('对象', '对象', None, []), - ('，', '，', None, []), - ('调', '调', None, []), - ('用', '用', None, []), - ('函数', '函数', None, []), - ('，', '，', None, []), - ('返回', '返回', None, []), - ('结果', '结果', None, []), - ('。', '。', None, []), - ('确保', '确保', None, []), - ('没有', '没有', None, []), - ('遗漏', '遗漏', None, []), - ('必要', '必要', None, []), - ('参数', '参数', None, []), - ('，', '，', None, []), - ('比如', '比如', None, []), - ('location', 'location', None, []), - ('是', '是', None, []), - ('必须', '必须', None, []), - ('的', '的', None, []), - ('，', '，', None, []), - ('这里', '这里', None, []), - ('已经', '已经', None, []), - ('提供', '提供', None, []), - ('，', '，', None, []), - ('所以', '所以', None, []), - ('没问题', '没问题', None, []), - ('。', '。', None, []), - ('最后', '最后', None, []), - ('将', '将', None, []), - ('结果', '结果', None, []), - ('以', '以', None, []), - ('自然', '自然', None, []), - ('语言', '语言', None, []), - ('回复', '回复', None, []), - ('用户', '用户', None, []), - ('。\n', '。\n', None, []), - ('', None, None, []), - ('\n\n', None, '\n\n', []), - ('', None, None, []), - ('\n', None, None, '\n'), - ('{"', None, None, '{"'), - ('name', None, None, 'name'), - ('":', None, None, '":'), - (' "', None, None, ' "'), - ('get', None, None, 'get'), - ('_weather', None, None, '_weather'), - ('",', None, None, '",'), - (' "', None, None, ' "'), - ('arguments', None, None, 'arguments'), - ('":', None, None, '":'), - (' {"', None, None, ' {"'), - ('location', None, None, 'location'), - ('":', None, None, '":'), - (' "', None, None, ' "'), - ('北京', None, None, '北京'), - ('"}}\n', None, None, '"}}\n'), - ('', None, None, None) -] - -DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS = DELTA_TEXT_SEQUENCE + [ - '\n\n', - '', - '\n', - '{"', - 'name', - '":', - ' "', - 'get', - '_weather', - '",', - ' "', - 'arguments', - '":', - ' {"', - 'location', - '":', - ' "', - '上海', - '"}}\n', - '', -] - -EXPECTED_CONTENT = '' -EXPECTED_REASONING_CONTENT = ''.join(( - '好的，用户问的是北京的天气怎么样。我需要调用get_weather这个工具来获取信息。', - '首先，确认用户提供的地点是北京，参数正确。然后检查工具的参数要求，', - '只需要location，类型是字符串。于是构造参数对象，调用函数，返回结果。', - '确保没有遗漏必要参数，比如location是必须的，这里已经提供，所以没问题。', - '最后将结果以自然语言回复用户。', -)) - - -def _normalize_delta_sequence(text_sequence: list) -> list[str]: - """Flatten streaming fixtures that use (delta, ...) tuples (possibly mixed - with str chunks).""" - if not text_sequence: - return [] - out = [] - for item in text_sequence: - out.append(item[0] if isinstance(item, tuple) else item) - return out - - -def _chat_completion_v1( - tokenizer: Tokenizer, - reasoning_parser: QwenReasoningParser, - tool_parser: Qwen3ToolParser, - request: ChatCompletionRequest, - text_sequence: list[str]) -> ChatCompletionResponse | Generator[ChatCompletionStreamResponse, None, None]: - request_id = f'chat-{shortuuid.random()}' - created_time = int(time.time()) - model_name = request.model - delta_chunks = _normalize_delta_sequence(text_sequence) - if request.stream: - parser_state = StreamBuffer() - has_parser = tool_parser is not None or reasoning_parser is not None - - def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, None, None]: - finish_reason = 'stop' - for text in delta_chunks: - print(f'delta_text: {text}') - # delta_message = DeltaMessage(role='assistant', content=None) - delta_message = DeltaMessage(role='assistant', content=text) if not has_parser else None - content = text - delta_token_ids = tokenizer.encode(content, add_bos=False) - parser_state.update(content, delta_token_ids) - if request.tool_choice != 'none' and tool_parser is not None: - delta_message = DeltaMessage(role='assistant') - tool_delta = tool_parser.extract_tool_calls_streaming( - delta_text=content, - delta_token_ids=delta_token_ids, - request=request, - stream_buffer=parser_state, - ) - print(f'tool_delta: {tool_delta}') - if tool_delta is not None: - delta_message.tool_calls = tool_delta.tool_calls - delta_message.content = tool_delta.content - if reasoning_parser is not None: - if tool_parser is None or delta_message is None: - content = text - elif delta_message.content is not None: - # delta_message.content is `content` if there is no tool call information in it - content = delta_message.content - # There might be reasoning content in `delta_message.content`. - # So we set it to None and let reasoning parser to extract the reasoning and content. - delta_message.content = None - else: - # tool_parser is consuming tool call information. We set Nont content to jump - # parsing reasoning. - content = None - reasoning_delta = reasoning_parser.extract_reasoning_streaming( - delta_text=content, - delta_token_ids=delta_token_ids, - request=request, - stream_buffer=parser_state, - ) - print(f'reasoning_delta: {reasoning_delta}') - if reasoning_delta is not None: - delta_message.reasoning_content = reasoning_delta.reasoning_content - delta_message.content = reasoning_delta.content - parser_state.step() - choice_data = ChatCompletionResponseStreamChoice(index=0, - delta=delta_message, - finish_reason=finish_reason) - response = ChatCompletionStreamResponse( - id=request_id, - created=created_time, - model=model_name, - choices=[choice_data] - ) - yield response - - return completion_stream_generator() - - # copied and simplified from api_server.py:chat_completions_v1 - text = ''.join(delta_chunks) - tool_calls = None - reasoning_content = None - finish_reason = 'stop' - if request.tool_choice != 'none' and tool_parser is not None: - tool_call_info = tool_parser.extract_tool_calls(text, request=request) - text, tool_calls = tool_call_info.content, tool_call_info.tool_calls - if isinstance(tool_calls, list) and len(tool_calls): - if finish_reason == 'stop': - finish_reason = 'tool_calls' - - if reasoning_parser is not None: - reasoning_content, text = reasoning_parser.extract_reasoning(text, request) - - choices = [] - choice_data = ChatCompletionResponseChoice( - index=0, - message=ChatMessage(role='assistant', content=text, tool_calls=tool_calls, reasoning_content=reasoning_content), - finish_reason=finish_reason, - ) - choices.append(choice_data) - - return ChatCompletionResponse( - id=request_id, - created=created_time, - model=model_name, - choices=choices, - usage=UsageInfo(), - ) - - -# def _stream_parse( -# tokenizer: Tokenizer, -# reasoning_parser: QwenReasoningParser, -# tool_parser: Qwen3ToolParser, -# request: ChatCompletionRequest, -# text_sequence: list[str], -# ) -> tuple[str, str, list[DeltaToolCall]]: -# # Call parser.extract_tool_calls_streaming with delta_text specified in `DELTA_TEXT_SEQUENCE`. -# # `current_text` and `previous_text` init values and update logic -# # can be found in lmdeploy/serve/openai/api_server.py:455-523. -# content = '' -# reasoning_content = '' -# tool_calls = {} - -# for stream_resp in _chat_completion_v1(tokenizer, reasoning_parser, tool_parser, request, text_sequence): -# delta_message: DeltaMessage = stream_resp.choices[0].delta -# if delta_message.content: -# content += delta_message.content -# if delta_message.reasoning_content: -# reasoning_content += delta_message.reasoning_content -# if delta_message.tool_calls: -# for c in delta_message.tool_calls: -# existing_call = tool_calls.get(c.id, None) -# if not existing_call: -# tool_calls[c.id] = c -# continue -# # merge with existing -# if c.function.name: -# existing_call.function.name = c.function.name -# if c.function.arguments: -# existing_call.function.arguments = existing_call.function.arguments or '' -# existing_call.function.arguments += c.function.arguments -# return content, reasoning_content, list(sorted(tool_calls.values(), key=lambda x: x.index)) - - - -class TestQwen3ToolStreamingParser: - """Tests for Qwen3ToolParser streaming mode.""" - - @pytest.mark.parametrize('text_sequence', [DELTA_TEXT_SEQUENCE]) - def test_parser_stream(self, tokenizer, reasoning_parser, tool_parser, - text_sequence: list[tuple[str, str, str, str]]): - """Test streaming parser with single and multiple tool calls.""" - request = ChatCompletionRequest(model='qwen', messages=[], stream=True) - delta_texts = [t[0] for t in text_sequence] - responses = _chat_completion_v1(tokenizer, reasoning_parser, tool_parser, request, delta_texts) - for response, t in zip(responses, text_sequence): - delta_message: DeltaMessage = response.choices[0].delta - print(f'delta_message: {delta_message}') - assert delta_message.reasoning_content == t[1] - assert delta_message.content == t[2] - # assert delta_message.tool_calls == t[3] - - - def test_incomplete_tool_call_streaming(self, tokenizer, reasoning_parser, tool_parser): - """Test streaming parser with incomplete tool call (missing end - tag).""" - request = ChatCompletionRequest(model='qwen', messages=[], stream=True) - - # Incomplete tool call without end tag - text_sequence = ['好的', '，', '让我', '调用', '工具', '。', 'Вот', '\n', 'ذهب', '\n', - '{"name": "get_weather", "arguments": {"location": "北京"'] - responses = _chat_completion_v1( - tokenizer, reasoning_parser, tool_parser, request, text_sequence) - for response in responses: - delta_message: DeltaMessage = response.choices[0].delta - print(f'delta_message: {delta_message}') - assert not delta_message.tool_calls - # Should not parse tool call since it's incomplete - - -class TestQwen3ToolNonStreamingParser: - """Tests for Qwen3ToolParser non-streaming mode.""" - - @pytest.mark.parametrize('text_sequence', [DELTA_TEXT_SEQUENCE, DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS]) - def test_parser_nonstream(self, tokenizer, reasoning_parser, tool_parser, text_sequence: list[str]): - """Test non-streaming parser with single and multiple tool calls.""" - full = ''.join(_normalize_delta_sequence(text_sequence)) - req = ChatCompletionRequest(model='qwen', messages=[], stream=False) - tool_ref = tool_parser.extract_tool_calls(full, request=req) - - resp: ChatCompletionResponse = _chat_completion_v1( - tokenizer, reasoning_parser, tool_parser, req, text_sequence) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert (first_message.content or '').strip() == EXPECTED_CONTENT - assert (first_message.reasoning_content or '').strip() == EXPECTED_REASONING_CONTENT - assert len(first_message.tool_calls) == len(tool_ref.tool_calls) - for parsed_call, ref_call in zip(first_message.tool_calls, tool_ref.tool_calls): - assert parsed_call.function.name == ref_call.function.name - assert json.loads(parsed_call.function.arguments) == json.loads(ref_call.function.arguments) - - def test_no_think_nonstream(self, tokenizer, reasoning_parser, tool_parser): - """Test non-streaming parser with plain text (no thinking tags).""" - text_sequence = [ - '你好', - '呀', - '！', - '✨', - '', - ' 很', - '高兴', - '见到', - '你', - '！', - ] - resp: ChatCompletionResponse = _chat_completion_v1( - tokenizer, reasoning_parser, tool_parser, - ChatCompletionRequest(model='qwen', messages=[], stream=False), - text_sequence) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert first_message.content == '你好呀！✨ 很高兴见到你！' - assert first_message.reasoning_content is None - - def test_invalid_json_tool_call(self, tokenizer, reasoning_parser, tool_parser): - """Test non-streaming parser with invalid JSON in tool call.""" - # Invalid JSON in tool call - text_sequence = ['好的，让我调用工具。', 'Вот', '\n', 'ذهب', '\n', - '{"name": "get_weather", "arguments": {invalid json}}', '666', '\n'] - - resp: ChatCompletionResponse = _chat_completion_v1( - tokenizer, reasoning_parser, tool_parser, - ChatCompletionRequest(model='qwen', messages=[], stream=False), - text_sequence) - - # Should handle gracefully - tool call may not be parsed due to invalid JSON - assert len(resp.choices) == 1 - - def test_empty_tool_call_content(self, tokenizer, reasoning_parser, tool_parser): - """Test non-streaming parser with empty tool call content.""" - # Empty tool call - text_sequence = ['好的', '。', 'Вот', '\n', 'ذهب', '\n', '666', '\n'] - - resp: ChatCompletionResponse = _chat_completion_v1( - tokenizer, reasoning_parser, tool_parser, - ChatCompletionRequest(model='qwen', messages=[], stream=False), - text_sequence) - - assert len(resp.choices) == 1 diff --git a/tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py b/tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py deleted file mode 100644 index 6061dee8dc..0000000000 --- a/tests/test_lmdeploy/server/tool_parsers/test_qwen3coder_parser.py +++ /dev/null @@ -1,410 +0,0 @@ -import collections -import json -import time -from collections.abc import Generator - -import pytest -import shortuuid - -from lmdeploy.model import MODELS -from lmdeploy.serve.openai.api_server import VariableInterface -from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - ChatCompletionResponse, - ChatCompletionResponseChoice, - ChatCompletionResponseStreamChoice, - ChatCompletionStreamResponse, - ChatMessage, - DeltaMessage, - DeltaToolCall, - UsageInfo, -) -from lmdeploy.serve.openai.response_parser import StreamBuffer -from lmdeploy.serve.openai.tool_parser.qwen3coder_tool_parser import Qwen3CoderToolParser - -TestExpects = collections.namedtuple('TestExpects', 'func_name kwargs') - - -class DummyTokenizer: - - def decode(self, token_ids: list[int]) -> str: - return ' '.join(map(str, token_ids)) - - def encode(self, text: str) -> list[int]: - return [ord(c) for c in text] - - -DELTA_TEXT_SEQUENCE = [ - '好的，我现在帮你调用工具。\n', - '', - '\n', - '\n', - '', - '北京\n', - 'celsius\n', - '\n', - '', -] - -DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS = DELTA_TEXT_SEQUENCE + [ - '\n\n', - '', - '\n\n', - '上海\n', - '\n', - '', -] - -EXPECTED_CONTENT = '好的，我现在帮你调用工具。' - - -def _chat_completion_v1( - request: ChatCompletionRequest, - text_sequence: list[str]) -> ChatCompletionResponse | Generator[ChatCompletionStreamResponse, None, None]: - request_id = f'chat-{shortuuid.random()}' - created_time = int(time.time()) - model_name = request.model - if request.stream: - - def completion_stream_generator() -> Generator[ChatCompletionStreamResponse, None, None]: - finish_reason = 'stop' - parser_state = StreamBuffer() - has_parser = (VariableInterface.tool_parser is not None or VariableInterface.reasoning_parser is not None) - for text in text_sequence: - logprobs, usage = None, None - delta_message = DeltaMessage(role='assistant', content=text) - if has_parser: - parser_state.update(text, []) - has_tool = VariableInterface.tool_parser is not None - if request.tool_choice != 'none' and has_tool: - tool_delta = VariableInterface.tool_parser.extract_tool_calls_streaming( - delta_text=text, - delta_token_ids=[], - request=request, - stream_buffer=parser_state, - ) - if tool_delta is not None: - delta_message.tool_calls = tool_delta.tool_calls - delta_message.content = tool_delta.content or '' - if VariableInterface.reasoning_parser is not None: - parser = VariableInterface.reasoning_parser - reasoning_delta = parser.extract_reasoning_streaming( - delta_text=delta_message.content, - delta_token_ids=[], - request=request, - stream_buffer=parser_state, - ) - if reasoning_delta is not None: - delta_message.reasoning_content = (reasoning_delta.reasoning_content) - delta_message.content = reasoning_delta.content or '' - if has_parser: - parser_state.step() - - choice_data = ChatCompletionResponseStreamChoice(index=0, - delta=delta_message, - finish_reason=finish_reason, - logprobs=logprobs) - response = ChatCompletionStreamResponse( - id=request_id, - created=created_time, - model=model_name, - choices=[choice_data], - usage=usage, - ) - yield response - - return completion_stream_generator() - - text = ''.join(text_sequence) - tool_calls = None - reasoning_content = None - finish_reason = 'stop' - has_tool = VariableInterface.tool_parser is not None - if request.tool_choice != 'none' and has_tool: - tool_call_info = VariableInterface.tool_parser.extract_tool_calls(text, request=request) - text, tool_calls = tool_call_info.content, tool_call_info.tool_calls - if isinstance(tool_calls, list) and len(tool_calls): - if finish_reason == 'stop': - finish_reason = 'tool_calls' - - if VariableInterface.reasoning_parser is not None: - parser = VariableInterface.reasoning_parser - reasoning_content, text = parser.extract_reasoning(text, request) - - choices = [] - choice_data = ChatCompletionResponseChoice( - index=0, - message=ChatMessage(role='assistant', content=text, tool_calls=tool_calls, reasoning_content=reasoning_content), - finish_reason=finish_reason, - ) - choices.append(choice_data) - - return ChatCompletionResponse( - id=request_id, - created=created_time, - model=model_name, - choices=choices, - usage=UsageInfo(), - ) - - -def _stream_parse(request: ChatCompletionRequest, text_sequence: list[str]) -> tuple[str, str, list[DeltaToolCall]]: - content = '' - reasoning_content = '' - tool_calls = {} - - for stream_resp in _chat_completion_v1(request, text_sequence): - delta_message: DeltaMessage = stream_resp.choices[0].delta - if delta_message.content: - content += delta_message.content - if delta_message.reasoning_content: - reasoning_content += delta_message.reasoning_content - if delta_message.tool_calls: - for c in delta_message.tool_calls: - existing_call = tool_calls.get(c.id, None) - if not existing_call: - tool_calls[c.id] = c - continue - # merge with existing - if c.function.name: - existing_call.function.name = c.function.name - if c.function.arguments: - existing_call.function.arguments = (existing_call.function.arguments or '') - existing_call.function.arguments += c.function.arguments - return content, reasoning_content, list(sorted(tool_calls.values(), key=lambda x: x.index)) - - -@pytest.mark.parametrize(('text_sequence', 'expects'), [ - (DELTA_TEXT_SEQUENCE, [TestExpects('get_weather', { - 'location': '北京', - 'unit': 'celsius' - })]), - (DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS, [ - TestExpects('get_weather', { - 'location': '北京', - 'unit': 'celsius' - }), - TestExpects('get_weather', {'location': '上海'}) - ]), -]) -def test_parser_stream(text_sequence: list[str], expects: list[TestExpects]): - tokenizer = DummyTokenizer() - VariableInterface.tool_parser = Qwen3CoderToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = None - request = ChatCompletionRequest(model='qwen3coder', messages=[], stream=True) - content, reasoning_content, tool_calls = _stream_parse(request, text_sequence) - assert len(tool_calls) == len(expects) - for parsed_call, expected_call in zip(tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args == expected_call.kwargs - assert content.strip() == EXPECTED_CONTENT - - -@pytest.mark.parametrize(('text_sequence', 'expects'), [ - (DELTA_TEXT_SEQUENCE, [TestExpects('get_weather', { - 'location': '北京', - 'unit': 'celsius' - })]), - (DELTA_TEXT_SEQUENCE_MULTIPLE_CALLS, [ - TestExpects('get_weather', { - 'location': '北京', - 'unit': 'celsius' - }), - TestExpects('get_weather', {'location': '上海'}) - ]), -]) -def test_parser_nonstream(text_sequence: list[str], expects: list[TestExpects]): - tokenizer = DummyTokenizer() - VariableInterface.tool_parser = Qwen3CoderToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = None - resp: ChatCompletionResponse = _chat_completion_v1( - ChatCompletionRequest(model='qwen3coder', messages=[], stream=False), text_sequence) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert first_message.content.strip() == EXPECTED_CONTENT - assert first_message.reasoning_content is None - assert len(first_message.tool_calls) == len(expects) - for parsed_call, expected_call in zip(first_message.tool_calls, expects): - assert parsed_call.function.name == expected_call.func_name - args = json.loads(parsed_call.function.arguments) - assert args == expected_call.kwargs - - -def test_no_think_nonstream(): - text_sequence = [ - '你好', - '呀', - '！', - '✨', - '', - ' 很', - '高兴', - '见到', - '你', - '！', - ] - tokenizer = DummyTokenizer() - VariableInterface.tool_parser = Qwen3CoderToolParser(tokenizer=tokenizer) - VariableInterface.reasoning_parser = None - resp: ChatCompletionResponse = _chat_completion_v1( - ChatCompletionRequest(model='qwen3coder', messages=[], stream=False), text_sequence) - - assert len(resp.choices) == 1 - first_message = resp.choices[0].message - assert first_message.content == '你好呀！✨ 很高兴见到你！' - assert first_message.reasoning_content is None - - -def test_adjust_request_parses_assistant_tool_call_object_arguments(): - parser = Qwen3CoderToolParser(tokenizer=DummyTokenizer()) - request = ChatCompletionRequest(model='qwen3coder', - messages=[{ - 'role': 'user', - 'content': 'hello' - }, { - 'role': 'assistant', - 'content': '', - 'tool_calls': [{ - 'id': 'call_1', - 'type': 'function', - 'function': { - 'name': 'get_weather', - 'arguments': '{"city": "Paris", "units": "metric"}' - } - }] - }]) - - adjusted_request = parser.adjust_request(request) - - assert adjusted_request is not request - assert adjusted_request.messages is not request.messages - assert adjusted_request.messages[1] is not request.messages[1] - assert adjusted_request.messages[1]['tool_calls'][0] is not request.messages[1]['tool_calls'][0] - assert adjusted_request.messages[1]['tool_calls'][0]['function']['arguments'] == { - 'city': 'Paris', - 'units': 'metric' - } - assert request.messages[1]['tool_calls'][0]['function']['arguments'] == '{"city": "Paris", "units": "metric"}' - - -@pytest.mark.parametrize('arguments', ['[1, 2, 3]', '1', '{not valid json}']) -def test_adjust_request_leaves_non_mapping_arguments_unchanged(arguments): - parser = Qwen3CoderToolParser(tokenizer=DummyTokenizer()) - request = ChatCompletionRequest(model='qwen3coder', - messages=[{ - 'role': 'assistant', - 'content': '', - 'tool_calls': [{ - 'id': 'call_1', - 'type': 'function', - 'function': { - 'name': 'fn', - 'arguments': arguments - } - }] - }]) - - adjusted_request = parser.adjust_request(request) - - assert adjusted_request is request - - -def test_adjust_request_noops_for_string_messages(): - parser = Qwen3CoderToolParser(tokenizer=DummyTokenizer()) - request = ChatCompletionRequest(model='qwen3coder', messages='hello') - - adjusted_request = parser.adjust_request(request) - - assert adjusted_request is request - - -def test_adjust_request_noops_without_assistant_tool_calls(): - parser = Qwen3CoderToolParser(tokenizer=DummyTokenizer()) - request = ChatCompletionRequest(model='qwen3coder', - messages=[{ - 'role': 'user', - 'content': 'hello' - }, { - 'role': 'assistant', - 'content': 'plain text response' - }, { - 'role': 'tool', - 'content': '', - 'tool_calls': [{ - 'id': 'call_1', - 'type': 'function', - 'function': { - 'name': 'fn', - 'arguments': '{"x": 1}' - } - }] - }]) - - adjusted_request = parser.adjust_request(request) - - assert adjusted_request is request - - -def test_adjust_request_noops_for_dict_arguments(): - parser = Qwen3CoderToolParser(tokenizer=DummyTokenizer()) - request = ChatCompletionRequest(model='qwen3coder', - messages=[{ - 'role': 'assistant', - 'content': '', - 'tool_calls': [{ - 'id': 'call_1', - 'type': 'function', - 'function': { - 'name': 'fn', - 'arguments': { - 'x': 1 - } - } - }] - }]) - - adjusted_request = parser.adjust_request(request) - - assert adjusted_request is request - - -@pytest.mark.parametrize('model_path', ['Qwen/Qwen3.5-35B-A3B']) -def test_adjust_request_renders_qwen_template_from_string_payload(model_path): - chat_template = MODELS.get('hf')(model_path) - parser = Qwen3CoderToolParser(tokenizer=DummyTokenizer()) - request = ChatCompletionRequest(model='qwen3coder', - messages=[{ - 'role': 'user', - 'content': 'What is the weather in Paris?' - }, { - 'role': 'assistant', - 'content': '', - 'tool_calls': [{ - 'id': 'call_1', - 'type': 'function', - 'function': { - 'name': 'get_weather', - 'arguments': '{"city":"Paris","units":"metric"}' - } - }] - }]) - - adjusted_request = parser.adjust_request(request) - prompt = chat_template.messages2prompt(adjusted_request.messages) - - assert adjusted_request is not request - assert adjusted_request.messages[1]['tool_calls'][0]['function']['arguments'] == { - 'city': 'Paris', - 'units': 'metric' - } - assert request.messages[1]['tool_calls'][0]['function']['arguments'] == '{"city":"Paris","units":"metric"}' - assert '' in prompt - assert '\nParis\n' in prompt - assert '\nmetric\n' in prompt From 754cf55cf58e5f3dd6f29f05b0037584ab4566b2 Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 10:31:46 +0000 Subject: [PATCH 10/14] the 4-th version --- .../serve/openai/reasoning_parser/__init__.py | 15 ----- .../deepseek_r1_reasoning_parser.py | 15 ----- .../deepseek_v3_reasoning_parser.py | 39 ------------ .../gpt_oss_reasoning_parser.py | 21 +------ .../identity_reasoning_parser.py | 30 --------- .../reasoning_parser/qwen_reasoning_parser.py | 19 ------ .../reasoning_parser/reasoning_parser.py | 63 +++---------------- lmdeploy/serve/openai/response_parser.py | 38 ++--------- .../server/parsers/test_qwen3_5_parsers.py | 6 +- .../server/parsers/test_qwen_parsers.py | 8 +-- 10 files changed, 21 insertions(+), 233 deletions(-) delete mode 100644 lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py delete mode 100644 lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py delete mode 100644 lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py delete mode 100644 lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py diff --git a/lmdeploy/serve/openai/reasoning_parser/__init__.py b/lmdeploy/serve/openai/reasoning_parser/__init__.py index 6e6f1072be..c6420377ad 100644 --- a/lmdeploy/serve/openai/reasoning_parser/__init__.py +++ b/lmdeploy/serve/openai/reasoning_parser/__init__.py @@ -1,27 +1,12 @@ # Copyright (c) OpenMMLab. All rights reserved. -from lmdeploy.serve.openai.response_parser import StreamBuffer - -from .deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from .deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser from .gpt_oss_reasoning_parser import GptOssReasoningParser -from .identity_reasoning_parser import IdentityReasoningParser -from .qwen_reasoning_parser import QwenReasoningParser from .reasoning_parser import ( ReasoningParser, ReasoningParserManager, - StreamingParserState, - ThinkingReasoningParser, ) __all__ = [ 'ReasoningParser', 'ReasoningParserManager', - 'StreamBuffer', - 'StreamingParserState', - 'ThinkingReasoningParser', - 'DeepSeekR1ReasoningParser', - 'QwenReasoningParser', - 'IdentityReasoningParser', - 'DeepSeekV3ReasoningParser', 'GptOssReasoningParser', ] diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py deleted file mode 100644 index b81e9da8cf..0000000000 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_r1_reasoning_parser.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -from .qwen_reasoning_parser import QwenReasoningParser -from .reasoning_parser import ReasoningParserManager - - -@ReasoningParserManager.register_module(name='deepseek-r1') -class DeepSeekR1ReasoningParser(QwenReasoningParser): - """Reasoning parser for DeepSeek R1 model. - - DeepSeek R1 always put tag to user's prompt. see more details in - https://huggingface.co/deepseek-ai/DeepSeek-R1/commit/8a58a132790c9935686eb97f042afa8013451c9f - Since DeepSeek-R1 and Qwen3-Thinking models have the same reasoning behavior, - we remove its original implementation and directly use QwenReasoningParser. - """ - pass diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py deleted file mode 100644 index 212a4d59a9..0000000000 --- a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. - -from typing import TYPE_CHECKING - -from .identity_reasoning_parser import IdentityReasoningParser -from .reasoning_parser import ReasoningParser - -if TYPE_CHECKING: - pass - -class DeepSeekV3ReasoningParser(ReasoningParser): - """The reasoning behavior of the DeepSeek V3.1 model varies depending on - the `enable_thinking` parameter. - - When set to True, a tag is added to the user's prompt, which corresponds to the thinking mode - of DeepSeek R1. - When `enable_thinking` is None, the thinking mode is disabled. In this case, the parser falls back to - the identity parser, which treats the entire model output as content and ignores any reasoning. - """ - - def __init__(self, tokenizer: object, **kwargs): - super().__init__(tokenizer, **kwargs) - - enable_thinking = bool(kwargs.get('enable_thinking', False)) - self._parser: ReasoningParser - if enable_thinking: - from .qwen_reasoning_parser import QwenReasoningParser as DeepSeekR1ReasoningParser - self._parser = DeepSeekR1ReasoningParser(tokenizer, **kwargs) - else: - self._parser = IdentityReasoningParser(tokenizer, **kwargs) - - def get_reasoning_open_tag(self) -> str | None: - return self._parser.get_reasoning_open_tag() - - def get_reasoning_close_tag(self) -> str | None: - return self._parser.get_reasoning_close_tag() - - def starts_in_reasoning_mode(self) -> bool: - return self._parser.starts_in_reasoning_mode() diff --git a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py index c43b7b1993..3cfc79d90c 100644 --- a/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/gpt_oss_reasoning_parser.py @@ -1,5 +1,4 @@ # Copyright (c) OpenMMLab. All rights reserved. -# Modified from https://github.com/vllm-project/vllm/blob/v0.10.2rc1/vllm/entrypoints/harmony_utils.py from __future__ import annotations import shortuuid @@ -31,8 +30,7 @@ def get_streamable_parser_for_assistant() -> StreamableParser: class GptOssChatParser: - """Harmony stream parser for GPT-OSS (assistant role): content, reasoning, - tool calls.""" + """Harmony stream parser for GPT-OSS (assistant role).""" def __init__(self): self.parser = get_streamable_parser_for_assistant() @@ -68,13 +66,7 @@ def parse_streaming(self, tokens: list[int]) -> DeltaMessage: index=base_index, function=DeltaFunctionCall(name=tool_name, arguments='')) elif delta_text: - # Continuing the same tool call. Ensure we don't duplicate the - # very first delta string in this chunk. Previously we initialized - # with arguments=delta_text and then appended again, causing - # duplicated content like "locationlocation". if delta_tool_call is None: - # We are in the middle of a tool call carried over from the - # previous chunk. Initialize an empty arguments buffer. delta_tool_call = DeltaToolCall(index=base_index, function=DeltaFunctionCall(arguments='')) delta_tool_call.function.arguments += delta_text @@ -101,25 +93,16 @@ def parse_full(self, tokens: list[int]) -> ChatMessage: @ReasoningParserManager.register_module('gpt-oss') class GptOssReasoningParser(ReasoningParser): - """Reasoning / channel parser for OpenAI Harmony GPT-OSS wire format (token - stream). - - Use ``--reasoning-parser gpt-oss`` when serving models that emit OpenAI Harmony - GPT-OSS token streams. - """ + """Reasoning / channel parser for OpenAI Harmony GPT-OSS wire format.""" def __init__(self, tokenizer: object, **kwargs): super().__init__(tokenizer, **kwargs) self._chat = GptOssChatParser() def parse_streaming(self, tokens: list[int]) -> DeltaMessage: - """Parse one engine chunk of token ids into a - :class:`~lmdeploy.serve.openai.protocol.DeltaMessage`.""" return self._chat.parse_streaming(tokens) def parse_full(self, tokens: list[int]) -> ChatMessage: - """Parse the full completion token sequence into a - :class:`~lmdeploy.serve.openai.protocol.ChatMessage`.""" return self._chat.parse_full(tokens) def get_reasoning_open_tag(self) -> str | None: diff --git a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py deleted file mode 100644 index 7ec8f65efc..0000000000 --- a/lmdeploy/serve/openai/reasoning_parser/identity_reasoning_parser.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. - -# modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/identity_reasoning_parser.py -from typing import TYPE_CHECKING - -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser - -if TYPE_CHECKING: - pass - - -class IdentityReasoningParser(ReasoningParser): - """Identity reasoning parser. - - This parser does not attempt to parse or strip out reasoning tokens. It treats the entire model output as content - and ignores reasoning. - """ - - def __init__(self, tokenizer, **kwargs): - super().__init__(tokenizer, **kwargs) - - - def get_reasoning_open_tag(self) -> str | None: - return None - - def get_reasoning_close_tag(self) -> str | None: - return None - - def starts_in_reasoning_mode(self) -> bool: - return False diff --git a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py deleted file mode 100644 index ab76e877bb..0000000000 --- a/lmdeploy/serve/openai/reasoning_parser/qwen_reasoning_parser.py +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) OpenMMLab. All rights reserved. -# modified from https://github.com/vllm-project/vllm/blob/main/vllm/reasoning/qwen3_reasoning_parser.py -from .reasoning_parser import ReasoningParserManager, ThinkingReasoningParser - - -@ReasoningParserManager.register_module(name=['qwen-qwq', 'qwen3', 'intern-s1', 'deepseeek-r1']) -class QwenReasoningParser(ThinkingReasoningParser): - """Reasoning parser for Qwen QwQ / Qwen3 / Intern-S / Qwen3.5 models. - - Qwen3 models, such as Qwen3-8B, Qwen3-**-Instruct, generate tag if enable_thinking is True. - However, Qwen3-Thinking models and Qwen3.5 models put in user's prompt, thus they don't - generate tag. Intern-S models hold the same behavior as Qwen3-Thinking models. - - This parser handles both styles: if appears in the generated output - it is stripped before extraction (non-streaming) or skipped (streaming). - """ - - start_token = '' - end_token = '' diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index cbcb769033..42fe8d1756 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -1,73 +1,24 @@ # Copyright (c) OpenMMLab. All rights reserved. -# modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/reasoning_parsers -from functools import cached_property - from mmengine import Registry -from lmdeploy.serve.openai.response_parser import StreamBuffer - ReasoningParserManager = Registry('reasoning_parser', locations=['lmdeploy.serve.openai.reasoning_parser']) -StreamingParserState = StreamBuffer - +@ReasoningParserManager.register_module(name=[ + 'qwen-qwq', 'qwen3', 'intern-s1', 'deepseek-r1', + 'deepseek-v3' +]) class ReasoningParser: - """Abstract base class for reasoning content parsers.""" + """Unified reasoning parser for all ``--reasoning-parser`` options.""" def __init__(self, tokenizer: object, **kwargs): self.model_tokenizer = tokenizer - @cached_property - def vocab(self) -> dict[str, int]: - # NOTE: Only PreTrainedTokenizerFast is guaranteed to have .vocab - # whereas all tokenizers have .get_vocab() - return self.model_tokenizer.get_vocab() - - def get_reasoning_open_tag(self) -> str | None: - """Return reasoning opening tag string, or None if no opening tag.""" - raise NotImplementedError('ReasoningParser.get_reasoning_open_tag has not been implemented!') - - def get_reasoning_close_tag(self) -> str | None: - """Return reasoning closing tag string, or None if no closing tag.""" - raise NotImplementedError('ReasoningParser.get_reasoning_close_tag has not been implemented!') - - def starts_in_reasoning_mode(self) -> bool: - """Whether streaming should begin in reasoning mode.""" - raise NotImplementedError('ReasoningParser.starts_in_reasoning_mode has not been implemented!') - - -class ThinkingReasoningParser(ReasoningParser): - """Base class for reasoning parsers that use ... style tags. - - Subclasses only need to set `start_token`, `end_token`. - - This parser uses a two-step detection strategy (inspired by vllm): - 1. First check token_ids (fast integer comparison) to determine whether - the start/end tags are present. - 2. Only when confirmed, use str.find() to locate exact positions for - slicing. - If the tokenizer does not have single-token representations for the tags, - it falls back to string-based detection automatically. - """ - - start_token: str = '' - end_token: str = '' - - - def __init__(self, tokenizer: object, **kwargs): - super().__init__(tokenizer, **kwargs) - - # Try to resolve single token ids for fast detection. - # If the tokenizer doesn't have them as single tokens, fall back to - # string-based detection (token ids will be None). - self.start_token_id: int = self.vocab.get(self.start_token) - self.end_token_id: int = self.vocab.get(self.end_token) - def get_reasoning_open_tag(self) -> str | None: - return self.start_token + return '' def get_reasoning_close_tag(self) -> str | None: - return self.end_token + return '' def starts_in_reasoning_mode(self) -> bool: return True diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py index b97a79a3f8..b8b7213e5b 100644 --- a/lmdeploy/serve/openai/response_parser.py +++ b/lmdeploy/serve/openai/response_parser.py @@ -2,7 +2,7 @@ """Unified profile-driven streaming parser for reasoning/content/tool calls.""" from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar from transformers import PreTrainedTokenizerBase @@ -22,25 +22,6 @@ logger = get_logger(__name__) -@dataclass -class StreamBuffer: - """Cumulative decode snapshot (``ResponseParser.stream_buffer``); also - passed as ``stream_buffer=``.""" - - previous_text: str = '' - current_text: str = '' - previous_token_ids: list[int] = field(default_factory=list) - current_token_ids: list[int] = field(default_factory=list) - - def update(self, delta_text: str, delta_token_ids: list[int]) -> None: - self.current_text += delta_text - self.current_token_ids.extend(delta_token_ids) - - def step(self) -> None: - self.previous_text = self.current_text - self.previous_token_ids = self.current_token_ids - - @dataclass class ProtocolProfile: reasoning_open_tag: str | None = None @@ -114,7 +95,7 @@ def __init__( self.request = self.tool_parser.adjust_request(request) else: self.request = request - self.stream_buffer = StreamBuffer() + self._accumulated_text = '' self.profile = self._build_profile() if (self.reasoning_parser is not None and self.enable_thinking is not False @@ -125,12 +106,6 @@ def __init__( self._pending = '' self._queued_deltas: list[_QueuedDelta] = [] - def _stream_update(self, delta_text: str, delta_token_ids: list[int]) -> None: - self.stream_buffer.update(delta_text, delta_token_ids) - - def _stream_step(self) -> None: - self.stream_buffer.step() - def stream_chunk( self, delta_text: str, @@ -155,15 +130,14 @@ def stream_chunk( if ( not delta_text and not delta_token_ids - and getattr(self, 'stream_buffer', None) is not None - and self.stream_buffer.current_text == '' + and self._accumulated_text == '' ): return DeltaMessage(role='assistant', content=''), False if self.tool_parser is None and self.reasoning_parser is None: return DeltaMessage(role='assistant', content=delta_text), False - self._stream_update(delta_text, delta_token_ids) + self._accumulated_text += delta_text self._pending += delta_text produced_any = False @@ -200,11 +174,9 @@ def stream_chunk( if ( delta_text == '' and not produced_any - and self.stream_buffer.current_text != '' + and self._accumulated_text != '' ): self._queued_deltas.append(_QueuedDelta(DeltaMessage(role='assistant', content=''), False)) - - self._stream_step() if not self._queued_deltas: return None, False queued = self._queued_deltas.pop(0) diff --git a/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py index 0142221c2d..7cf921ae6d 100644 --- a/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py +++ b/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py @@ -1,7 +1,7 @@ import pytest from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaToolCall -from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser from lmdeploy.serve.openai.response_parser import ResponseParser from lmdeploy.serve.openai.tool_parser.qwen3coder_tool_parser import Qwen3CoderToolParser from lmdeploy.tokenizer import HuggingFaceTokenizer @@ -19,8 +19,8 @@ def tokenizer(): @pytest.fixture() def response_parser(tokenizer): - # Configure ResponseParser to use Qwen3 reasoning parser and Qwen3.5 Coder tool parser. - ResponseParser.reasoning_parser_cls = QwenReasoningParser + # Configure ResponseParser to use unified reasoning parser and Qwen3.5 Coder tool parser. + ResponseParser.reasoning_parser_cls = ReasoningParser ResponseParser.tool_parser_cls = Qwen3CoderToolParser request = ChatCompletionRequest( diff --git a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py index 3d9246c6c9..bd8109e294 100644 --- a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py +++ b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py @@ -1,7 +1,7 @@ import pytest from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaToolCall -from lmdeploy.serve.openai.reasoning_parser.qwen_reasoning_parser import QwenReasoningParser +from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser from lmdeploy.serve.openai.response_parser import ResponseParser from lmdeploy.serve.openai.tool_parser.qwen3_tool_parser import Qwen3ToolParser from lmdeploy.tokenizer import HuggingFaceTokenizer @@ -19,8 +19,8 @@ def tokenizer(): @pytest.fixture() def response_parser(tokenizer): - # Configure ResponseParser to use Qwen3 reasoning and tool parsers. - ResponseParser.reasoning_parser_cls = QwenReasoningParser + # Configure ResponseParser to use unified reasoning parser and Qwen3 tool parser. + ResponseParser.reasoning_parser_cls = ReasoningParser ResponseParser.tool_parser_cls = Qwen3ToolParser request = ChatCompletionRequest( @@ -346,7 +346,7 @@ def _call(delta_text: str): def test_stream_chunk_preserves_content_reasoning_content_order(self, tokenizer, response_parser): """Mixed single chunk should preserve event order without content merge.""" - class PlainStartQwenReasoningParser(QwenReasoningParser): + class PlainStartQwenReasoningParser(ReasoningParser): def starts_in_reasoning_mode(self) -> bool: return False From 39ca371295eec7e82b8f38c7dcacf3fd171c01ce Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 12:43:11 +0000 Subject: [PATCH 11/14] fix --- lmdeploy/serve/openai/api_server.py | 12 +- lmdeploy/serve/openai/harmony_utils.py | 14 +- .../serve/openai/tool_parser/tool_parser.py | 76 +--------- .../parsers/test_gpt_oss_reasoning_parser.py | 131 ++++++++++++++++++ .../server/parsers/test_qwen_parsers.py | 12 +- 5 files changed, 161 insertions(+), 84 deletions(-) create mode 100644 tests/test_lmdeploy/server/parsers/test_gpt_oss_reasoning_parser.py diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 97d38c95b9..7a5c19e10e 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -483,8 +483,18 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: if res.finish_reason == 'stop' and streaming_tools is True: res.finish_reason = 'tool_calls' elif request.tool_choice != 'none' and request.tools is not None: - if ResponseParser.tool_parser is None: + if ResponseParser.tool_parser_cls is None: logger.error('Please launch the api_server with --tool-call-parser if you want to use tool.') + + # The parser may intentionally suppress no-op chunks by returning + # ``None``. Keep them suppressed unless this is a visible terminal + # frame (finish/usage/logprobs), where OpenAI-style streams still + # expect a delta object. + if delta_message is None: + if res.finish_reason is None and usage is None and logprobs is None: + continue + delta_message = DeltaMessage(role='assistant') + if request.return_token_ids: delta_message.gen_tokens = delta_token_ids response_json = create_stream_response_json(index=0, diff --git a/lmdeploy/serve/openai/harmony_utils.py b/lmdeploy/serve/openai/harmony_utils.py index 1b35aa8eff..2024517c9d 100644 --- a/lmdeploy/serve/openai/harmony_utils.py +++ b/lmdeploy/serve/openai/harmony_utils.py @@ -1,16 +1,14 @@ # Copyright (c) OpenMMLab. All rights reserved. -"""Backward-compatible re-exports for Harmony GPT-OSS helpers. +"""Backward-compatibility shim for GPT-OSS Harmony parser. -Prefer importing from :mod:`lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser`. +The canonical implementation now lives in: +`lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser`. """ -from lmdeploy.serve.openai.reasoning_parser.gpt_oss_reasoning_parser import ( + +from .reasoning_parser.gpt_oss_reasoning_parser import ( # noqa: F401 GptOssChatParser, get_encoding, get_streamable_parser_for_assistant, ) -__all__ = [ - 'GptOssChatParser', - 'get_encoding', - 'get_streamable_parser_for_assistant', -] +__all__ = ['GptOssChatParser', 'get_encoding', 'get_streamable_parser_for_assistant'] diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index bafa91242a..85c795a269 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -131,85 +131,23 @@ def _decode_tool_incremental_json(self, added_text: str, *, final: bool) -> list if args_obj is None: return out - if isinstance(args_obj, dict): - items = list(args_obj.items()) - if not self._args_prefix_emitted and items: - first_key = items[0][0] - out.append( - DeltaToolCall( - id=self._active_tool_call_id, - index=self._active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=f'{{\"{first_key}\": \"')), - ) - self._args_prefix_emitted = True - - values_concat = ''.join(v for _, v in items if isinstance(v, str)) - if len(values_concat) > self._value_chars_emitted: - diff = values_concat[self._value_chars_emitted:] - out.append( - DeltaToolCall( - id=self._active_tool_call_id, - index=self._active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=diff), - )) - self._value_chars_emitted = len(values_concat) - - if self._is_complete_json(payload) and self._args_prefix_emitted and not self._args_closed_emitted: - out.append( - DeltaToolCall( - id=self._active_tool_call_id, - index=self._active_tool_index, - type=None, - function=DeltaFunctionCall(arguments='"}'), - )) - self._args_closed_emitted = True - return out - args_json = json.dumps(args_obj, ensure_ascii=False) if args_json in ('{}', '[]'): return out - emitted_arg = False - candidate: str | None = None - if self._is_complete_json(payload): - candidate = args_json - elif self._prev_args_json: - candidate = self._common_prefix(self._prev_args_json, args_json) - elif self._args_emitted_len == 0 and added_text: - pos = args_json.find(added_text) - if pos >= 0: - candidate = args_json[:pos + len(added_text)] - - if candidate and len(candidate) > self._args_emitted_len: - diff = candidate[self._args_emitted_len:] - if final or any(ch.isalnum() for ch in diff): - out.append( - DeltaToolCall( - id=self._active_tool_call_id, - index=self._active_tool_index, - type=None, - function=DeltaFunctionCall(arguments=diff), - )) - self._args_emitted_len = len(candidate) - emitted_arg = True - - if ( - not emitted_arg - and self._args_emitted_len > 0 - and added_text - and any(ord(ch) > 127 for ch in added_text) - ): + # Emit argument text only when the tool payload is complete. This keeps + # streamed argument chunks valid JSON and avoids malformed intermediate + # fragments when partial parsers expose transient dict states. + if final and len(args_json) > self._args_emitted_len: + diff = args_json[self._args_emitted_len:] out.append( DeltaToolCall( id=self._active_tool_call_id, index=self._active_tool_index, type=None, - function=DeltaFunctionCall(arguments=added_text), + function=DeltaFunctionCall(arguments=diff), )) - self._args_emitted_len += len(added_text) - self._prev_args_json = args_json + self._args_emitted_len = len(args_json) return out @staticmethod diff --git a/tests/test_lmdeploy/server/parsers/test_gpt_oss_reasoning_parser.py b/tests/test_lmdeploy/server/parsers/test_gpt_oss_reasoning_parser.py new file mode 100644 index 0000000000..680dabb416 --- /dev/null +++ b/tests/test_lmdeploy/server/parsers/test_gpt_oss_reasoning_parser.py @@ -0,0 +1,131 @@ +from dataclasses import dataclass + +from lmdeploy.serve.openai.reasoning_parser import gpt_oss_reasoning_parser as gpt_oss_mod + + +@dataclass +class _FakeMsg: + channel: str + recipient: str | None + + +class _FakeStreamableParser: + """A tiny scripted parser to emulate openai_harmony.StreamableParser.""" + + def __init__(self, script: dict[int, dict]): + self._script = script + self.current_channel = 'final' + self.current_recipient = None + self.last_content_delta = '' + self.messages: list[_FakeMsg] = [] + + def process(self, token: int): + event = self._script[token] + next_channel = event['channel'] + next_recipient = event.get('recipient') + + # Mirror completed function-call message accounting used by the parser + # to compute tool call index. + if (self.current_channel == 'commentary' and self.current_recipient + and self.current_recipient.startswith('functions.') and next_recipient != self.current_recipient): + self.messages.append(_FakeMsg(channel='commentary', recipient=self.current_recipient)) + + self.current_channel = next_channel + self.current_recipient = next_recipient + self.last_content_delta = event.get('delta', '') + + +def _scripted_events() -> dict[int, dict]: + return { + 1: { + 'channel': 'analysis', + 'recipient': None, + 'delta': 'Need tool. ', + }, + 2: { + 'channel': 'commentary', + 'recipient': 'functions.get_weather', + 'delta': '', + }, + 3: { + 'channel': 'commentary', + 'recipient': 'functions.get_weather', + 'delta': '{"location":"', + }, + 4: { + 'channel': 'commentary', + 'recipient': 'functions.get_weather', + 'delta': 'Beijing"}', + }, + 5: { + 'channel': 'commentary', + 'recipient': 'functions.get_time', + 'delta': '', + }, + 6: { + 'channel': 'commentary', + 'recipient': 'functions.get_time', + 'delta': '{"tz":"UTC"}', + }, + 7: { + 'channel': 'final', + 'recipient': None, + 'delta': 'Result: ', + }, + 8: { + 'channel': 'final', + 'recipient': None, + 'delta': 'sunny', + }, + } + + +def test_gpt_oss_chat_parser_routes_channels(monkeypatch): + monkeypatch.setattr( + gpt_oss_mod, + 'get_streamable_parser_for_assistant', + lambda: _FakeStreamableParser(_scripted_events()), + ) + + parser = gpt_oss_mod.GptOssChatParser() + delta = parser.parse_streaming([1, 2, 3, 4, 5, 6, 7, 8]) + + assert delta.content == 'Result: sunny' + assert delta.reasoning_content == 'Need tool. ' + assert delta.tool_calls is not None + assert len(delta.tool_calls) == 2 + + first, second = delta.tool_calls + assert first.function is not None + assert first.function.name == 'get_weather' + assert first.function.arguments == '{"location":"Beijing"}' + assert first.index == 0 + + assert second.function is not None + assert second.function.name == 'get_time' + assert second.function.arguments == '{"tz":"UTC"}' + assert second.index == 1 + + +def test_gpt_oss_reasoning_parser_parse_full(monkeypatch): + monkeypatch.setattr( + gpt_oss_mod, + 'get_streamable_parser_for_assistant', + lambda: _FakeStreamableParser(_scripted_events()), + ) + + parser = gpt_oss_mod.GptOssReasoningParser(tokenizer=object()) + message = parser.parse_full([1, 2, 3, 4, 5, 6, 7, 8]) + + assert message.content == 'Result: sunny' + assert message.reasoning_content == 'Need tool. ' + assert message.tool_calls is not None + assert [call.function.name for call in message.tool_calls] == ['get_weather', 'get_time'] + assert [call.function.arguments for call in message.tool_calls] == ['{"location":"Beijing"}', '{"tz":"UTC"}'] + + +def test_gpt_oss_reasoning_parser_tags(): + parser = gpt_oss_mod.GptOssReasoningParser(tokenizer=object()) + assert parser.get_reasoning_open_tag() is None + assert parser.get_reasoning_close_tag() is None + assert parser.starts_in_reasoning_mode() is False diff --git a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py index bd8109e294..6ef2707f3a 100644 --- a/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py +++ b/tests/test_lmdeploy/server/parsers/test_qwen_parsers.py @@ -139,16 +139,16 @@ def response_parser(tokenizer): (' {"', False, None, None, False, None, None, None), ('location', False, None, None, False, None, None, None), ('":', False, None, None, False, None, None, None), - (' "', True, None, None, True, None, '{"location": "', None), - ('北京', True, None, None, True, None, '北京', None), - ('",', False, None, None, True, None, '",', None), + (' "', False, None, None, False, None, None, None), + ('北京', False, None, None, False, None, None, None), + ('",', False, None, None, False, None, None, None), (' "', False, None, None, False, None, None, None), ('unit', False, None, None, False, None, None, None), ('":', False, None, None, False, None, None, None), (' "', False, None, None, False, None, None, None), - ('celsius', True, None, None, True, None, 'celsius', None), - ('"}}\n', True, None, None, True, None, '"}', None), - ('', False, None, None, False, None, None, None), + ('celsius', False, None, None, False, None, None, None), + ('"}}\n', False, None, None, False, None, None, None), + ('', True, None, None, True, None, '{"location": "北京", "unit": "celsius"}', None), ('', True, None, '', False, None, None, None), ] From 525eb871601202bffc564d5910ddc13b9fff96ec Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 13:36:00 +0000 Subject: [PATCH 12/14] type hint --- .../serve/openai/reasoning_parser/__init__.py | 2 + .../deepseek_v3_reasoning_parser.py | 20 +++++++++ .../reasoning_parser/reasoning_parser.py | 12 +++-- lmdeploy/serve/openai/response_parser.py | 28 ++++++++---- .../tool_parser/internlm2_tool_parser.py | 6 ++- .../tool_parser/qwen3coder_tool_parser.py | 8 +++- .../serve/openai/tool_parser/tool_parser.py | 7 ++- .../test_deepseek_v3_reasoning_parser.py | 45 +++++++++++++++++++ 8 files changed, 112 insertions(+), 16 deletions(-) create mode 100644 lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py create mode 100644 tests/test_lmdeploy/server/parsers/test_deepseek_v3_reasoning_parser.py diff --git a/lmdeploy/serve/openai/reasoning_parser/__init__.py b/lmdeploy/serve/openai/reasoning_parser/__init__.py index c6420377ad..eb8550e710 100644 --- a/lmdeploy/serve/openai/reasoning_parser/__init__.py +++ b/lmdeploy/serve/openai/reasoning_parser/__init__.py @@ -1,4 +1,5 @@ # Copyright (c) OpenMMLab. All rights reserved. +from .deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser from .gpt_oss_reasoning_parser import GptOssReasoningParser from .reasoning_parser import ( ReasoningParser, @@ -8,5 +9,6 @@ __all__ = [ 'ReasoningParser', 'ReasoningParserManager', + 'DeepSeekV3ReasoningParser', 'GptOssReasoningParser', ] diff --git a/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py new file mode 100644 index 0000000000..93bb6e64c9 --- /dev/null +++ b/lmdeploy/serve/openai/reasoning_parser/deepseek_v3_reasoning_parser.py @@ -0,0 +1,20 @@ +# Copyright (c) OpenMMLab. All rights reserved. +from .reasoning_parser import ReasoningParser, ReasoningParserManager + + +@ReasoningParserManager.register_module('deepseek-v3') +class DeepSeekV3ReasoningParser(ReasoningParser): + """Reasoning parser for DeepSeek-V3. + + DeepSeek-V3 differs from qwen3 default behavior: + - ``enable_thinking=True``: model can emit reasoning stream (...) + - ``enable_thinking=None``: model typically emits no reasoning part + """ + + def __init__(self, tokenizer: object, **kwargs): + super().__init__(tokenizer, **kwargs) + self.enable_thinking = kwargs.get('enable_thinking', None) + + def starts_in_reasoning_mode(self) -> bool: + # Enter reasoning mode only when explicitly requested. + return self.enable_thinking is True diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index 42fe8d1756..d4165da920 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -1,17 +1,23 @@ # Copyright (c) OpenMMLab. All rights reserved. +from __future__ import annotations + +from typing import TYPE_CHECKING + from mmengine import Registry +if TYPE_CHECKING: + from transformers import PreTrainedTokenizerBase + ReasoningParserManager = Registry('reasoning_parser', locations=['lmdeploy.serve.openai.reasoning_parser']) @ReasoningParserManager.register_module(name=[ - 'qwen-qwq', 'qwen3', 'intern-s1', 'deepseek-r1', - 'deepseek-v3' + 'qwen-qwq', 'qwen3', 'intern-s1', 'deepseek-r1' ]) class ReasoningParser: """Unified reasoning parser for all ``--reasoning-parser`` options.""" - def __init__(self, tokenizer: object, **kwargs): + def __init__(self, tokenizer: PreTrainedTokenizerBase, **kwargs): self.model_tokenizer = tokenizer def get_reasoning_open_tag(self) -> str | None: diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py index b8b7213e5b..5468cc1174 100644 --- a/lmdeploy/serve/openai/response_parser.py +++ b/lmdeploy/serve/openai/response_parser.py @@ -5,25 +5,35 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, ClassVar -from transformers import PreTrainedTokenizerBase - -from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, - DeltaMessage, - DeltaToolCall, - ToolCall, -) +from lmdeploy.serve.openai.protocol import DeltaMessage from lmdeploy.utils import get_logger if TYPE_CHECKING: + from transformers import PreTrainedTokenizerBase + + from lmdeploy.serve.openai.protocol import ChatCompletionRequest, DeltaToolCall, ToolCall from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParser from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParser -logger = get_logger(__name__) +logger = get_logger('lmdeploy') @dataclass class ProtocolProfile: + """Protocol tags and startup mode used by :class:`ResponseParser`. + + ``starts_in_reasoning_mode`` decides the initial parse mode before any tags are seen. + In ResponseParser, it controls whether the parser treats the beginning of generation as: + - reasoning (MODE_REASONING) -> text goes to reasoning_content, or + - plain (MODE_PLAIN) -> text goes to normal content. + Practically: + - If parser has reasoning support, ``enable_thinking`` is not False, and + ``starts_in_reasoning_mode=True``, first chunks are parsed as reasoning until ````. + - Otherwise it starts in plain mode and only enters reasoning when it sees ````. + It is only a profile default and can be customized by concrete reasoning + parsers (for example DeepSeek-V3). + """ + reasoning_open_tag: str | None = None reasoning_close_tag: str | None = None tool_open_tag: str | None = None diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index 5b804d5518..82fcd7243a 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -1,8 +1,9 @@ # Copyright (c) OpenMMLab. All rights reserved. +from __future__ import annotations +from typing import TYPE_CHECKING from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, DeltaToolCall, ToolCall, ) @@ -10,6 +11,9 @@ from .tool_parser import ToolParser, ToolParserManager +if TYPE_CHECKING: + from lmdeploy.serve.openai.protocol import ChatCompletionRequest + logger = get_logger('lmdeploy') diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index a44498cd3b..94207e1c22 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -1,10 +1,11 @@ # Copyright (c) OpenMMLab. All rights reserved. +from __future__ import annotations + import json import re -from typing import Any +from typing import TYPE_CHECKING, Any from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, DeltaFunctionCall, DeltaToolCall, FunctionCall, @@ -14,6 +15,9 @@ from .tool_parser import ToolParser, ToolParserManager +if TYPE_CHECKING: + from lmdeploy.serve.openai.protocol import ChatCompletionRequest + logger = get_logger('lmdeploy') diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index 85c795a269..6d6f5f800e 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -1,7 +1,10 @@ # Copyright (c) OpenMMLab. All rights reserved. # modified from https://github.com/vllm-project/vllm/tree/v0.7.3/vllm/entrypoints/openai/tool_parsers +from __future__ import annotations + import json from functools import cached_property +from typing import TYPE_CHECKING import partial_json_parser import shortuuid @@ -9,7 +12,6 @@ from partial_json_parser.core.options import Allow from lmdeploy.serve.openai.protocol import ( - ChatCompletionRequest, DeltaFunctionCall, DeltaToolCall, FunctionCall, @@ -17,6 +19,9 @@ ) from lmdeploy.utils import get_logger +if TYPE_CHECKING: + from lmdeploy.serve.openai.protocol import ChatCompletionRequest + logger = get_logger('lmdeploy') ToolParserManager = Registry('tool_parser', locations=['lmdeploy.serve.openai.tool_parser']) diff --git a/tests/test_lmdeploy/server/parsers/test_deepseek_v3_reasoning_parser.py b/tests/test_lmdeploy/server/parsers/test_deepseek_v3_reasoning_parser.py new file mode 100644 index 0000000000..d209ef806d --- /dev/null +++ b/tests/test_lmdeploy/server/parsers/test_deepseek_v3_reasoning_parser.py @@ -0,0 +1,45 @@ +from lmdeploy.serve.openai.protocol import ChatCompletionRequest +from lmdeploy.serve.openai.response_parser import ResponseParser + + +def _make_parser(enable_thinking): + from lmdeploy.serve.openai.reasoning_parser.deepseek_v3_reasoning_parser import DeepSeekV3ReasoningParser + + old_reasoning_cls = ResponseParser.reasoning_parser_cls + old_tool_cls = ResponseParser.tool_parser_cls + ResponseParser.reasoning_parser_cls = DeepSeekV3ReasoningParser + ResponseParser.tool_parser_cls = None + request = ChatCompletionRequest( + model='deepseek-v3', + messages=[], + stream=True, + chat_template_kwargs={'enable_thinking': enable_thinking}, + ) + parser = ResponseParser(request=request, tokenizer=object()) + return parser, old_reasoning_cls, old_tool_cls + + +def test_deepseek_v3_starts_plain_when_enable_thinking_none(): + parser, old_reasoning_cls, old_tool_cls = _make_parser(enable_thinking=None) + try: + delta_msg, tool_emitted = parser.stream_chunk(delta_text='hello', delta_token_ids=[]) + assert tool_emitted is False + assert delta_msg is not None + assert delta_msg.content == 'hello' + assert delta_msg.reasoning_content is None + finally: + ResponseParser.reasoning_parser_cls = old_reasoning_cls + ResponseParser.tool_parser_cls = old_tool_cls + + +def test_deepseek_v3_starts_reasoning_when_enable_thinking_true(): + parser, old_reasoning_cls, old_tool_cls = _make_parser(enable_thinking=True) + try: + delta_msg, tool_emitted = parser.stream_chunk(delta_text='hello', delta_token_ids=[]) + assert tool_emitted is False + assert delta_msg is not None + assert delta_msg.content is None + assert delta_msg.reasoning_content == 'hello' + finally: + ResponseParser.reasoning_parser_cls = old_reasoning_cls + ResponseParser.tool_parser_cls = old_tool_cls From dd1280bd8632f96f3b8136a13dc203e23739e6d7 Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Wed, 1 Apr 2026 13:53:00 +0000 Subject: [PATCH 13/14] remove unused code --- .../tool_parser/internlm2_tool_parser.py | 33 +++------- .../openai/tool_parser/llama3_tool_parser.py | 20 +------ .../openai/tool_parser/qwen2d5_tool_parser.py | 20 +------ .../openai/tool_parser/qwen3_tool_parser.py | 60 +------------------ .../tool_parser/qwen3coder_tool_parser.py | 41 ++----------- .../serve/openai/tool_parser/tool_parser.py | 35 +---------- .../server/parsers/test_qwen3_5_parsers.py | 19 ++++++ 7 files changed, 40 insertions(+), 188 deletions(-) diff --git a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py index 82fcd7243a..a980d393d0 100644 --- a/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/internlm2_tool_parser.py @@ -3,30 +3,23 @@ from typing import TYPE_CHECKING -from lmdeploy.serve.openai.protocol import ( - DeltaToolCall, - ToolCall, -) -from lmdeploy.utils import get_logger - from .tool_parser import ToolParser, ToolParserManager if TYPE_CHECKING: - from lmdeploy.serve.openai.protocol import ChatCompletionRequest - -logger = get_logger('lmdeploy') + from transformers import PreTrainedTokenizerBase + from lmdeploy.serve.openai.protocol import ( + ChatCompletionRequest, + DeltaToolCall, + ToolCall, + ) @ToolParserManager.register_module(['internlm', 'intern-s1']) class Internlm2ToolParser(ToolParser): + """Tool parser for InternLM JSON tool-call payloads.""" - def __init__(self, tokenizer: object): + def __init__(self, tokenizer: PreTrainedTokenizerBase): super().__init__(tokenizer) - self.parse_cursor = 0 - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.streamed_args_for_tool: list[str] = [] - self.prev_tool_call_arr: list[dict] = [] def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: if request.tools and request.tool_choice != 'none': @@ -36,13 +29,6 @@ def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionReques request.skip_special_tokens = False return request - def get_argments(self, obj): - if 'parameters' in obj: - return obj.get('parameters') - elif 'arguments' in obj: - return obj.get('arguments') - return None - def get_tool_open_tag(self) -> str | None: return '<|action_start|><|plugin|>' @@ -53,8 +39,7 @@ def get_tool_payload_format(self) -> str: return 'json' def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: - """InternLM2 tool payload is JSON; reuse shared JSON incremental - decoder.""" + """Decode incremental JSON tool payload.""" return self._decode_tool_incremental_json(added_text=added_text, final=final) def parse_tool_call_complete(self, payload: str) -> ToolCall | None: diff --git a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py index 29d091fa0e..04b23fff16 100644 --- a/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/llama3_tool_parser.py @@ -1,35 +1,20 @@ # Copyright (c) OpenMMLab. All rights reserved. -import re from lmdeploy.serve.openai.protocol import ( DeltaToolCall, ToolCall, ) -from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -logger = get_logger('lmdeploy') - @ToolParserManager.register_module('llama3') class Llama3JsonToolParser(ToolParser): - """Tool call parser for Llama 3.1 models intended for use with the - examples/tool_chat_template_llama.jinja template. - - Used when --tool-call-parser llama3 are all set - """ + """Tool parser for Llama3 JSON tool-call payloads.""" def __init__(self, tokenizer: object): super().__init__(tokenizer) - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.streamed_args_for_tool: list[str] = [] - self.prev_tool_call_arr: list[dict] = [] - self.bot_token = '<|python_tag|>' - self.bot_token_id = tokenizer.encode(self.bot_token, add_special_tokens=False)[0] - self.tool_call_regex = re.compile(r'\[{.*?}\]', re.DOTALL) def get_tool_open_tag(self) -> str | None: return self.bot_token @@ -41,8 +26,7 @@ def get_tool_payload_format(self) -> str: return 'json' def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: - """Llama3 tool payload is JSON; reuse shared JSON incremental - decoder.""" + """Decode incremental JSON tool payload.""" return self._decode_tool_incremental_json(added_text=added_text, final=final) def parse_tool_call_complete(self, payload: str) -> ToolCall | None: diff --git a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py index 35cbb95449..bdaa45a1f5 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen2d5_tool_parser.py @@ -5,33 +5,18 @@ DeltaToolCall, ToolCall, ) -from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -logger = get_logger('lmdeploy') - @ToolParserManager.register_module(['qwen2d5']) class Qwen2d5ToolParser(ToolParser): + """Tool parser for Qwen2.5 JSON tool-call payloads.""" def __init__(self, tokenizer: object): super().__init__(tokenizer) self.tool_start_token = '' self.tool_end_token = '' - self.pattern = r'(.*?)' - self.parse_cursor = 0 - self.current_tool_id = -1 - self.current_tool_name_sent = False - self.streamed_args_for_tool: list[str] = [] - self.prev_tool_call_arr: list[dict] = [] - - def get_argments(self, obj): - if 'parameters' in obj: - return obj.get('parameters') - elif 'arguments' in obj: - return obj.get('arguments') - return None def get_tool_open_tag(self) -> str | None: return self.tool_start_token @@ -43,8 +28,7 @@ def get_tool_payload_format(self) -> str: return 'json' def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: - """Qwen2.5 tool payload is JSON; reuse shared JSON incremental - decoder.""" + """Decode incremental JSON tool payload.""" return self._decode_tool_incremental_json(added_text=added_text, final=final) def parse_tool_call_complete(self, payload: str) -> ToolCall | None: diff --git a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py index bb72ed1896..58a2189616 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3_tool_parser.py @@ -1,49 +1,21 @@ # Copyright (c) OpenMMLab. All rights reserved. -import re from lmdeploy.serve.openai.protocol import ( DeltaToolCall, ToolCall, ) -from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager -logger = get_logger('lmdeploy') - @ToolParserManager.register_module(['qwen', 'qwen3']) class Qwen3ToolParser(ToolParser): - """Parser for Qwen3 model's tool call format. - - Handles the extraction of tool calls from Qwen3's output format, which uses XML-like tags for tool calls and - reasoning. - """ + """Tool parser for Qwen3 JSON tool-call payloads.""" def __init__(self, tokenizer: object): super().__init__(tokenizer) self.tool_start_token = '' self.tool_end_token = '' - self.tool_call_pattern = re.compile(r'\n*(.*?)', re.DOTALL) - self.parse_cursor = 0 - self.qwen_tool_serial_index = -1 - self.qwen_active_tool_call_id = '' - self.current_tool_name_sent = False - self.prev_tool_call_arr: list[dict] = [] - self.streamed_args_for_tool: list[str] = [] - # True when we are between and in the accumulated output. - self.in_tool_block: bool = False - - def get_argments(self, obj): - """Extract arguments from tool call object, handling different formats. - - Supports both 'parameters' and 'arguments' keys in the tool call object. - """ - if 'parameters' in obj: - return obj.get('parameters') - elif 'arguments' in obj: - return obj.get('arguments') - return None def get_tool_open_tag(self) -> str | None: return self.tool_start_token @@ -55,36 +27,8 @@ def get_tool_payload_format(self) -> str: return 'json' def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: - """Decode Qwen3 JSON tool payload incrementally.""" + """Decode incremental JSON tool payload.""" return self._decode_tool_incremental_json(added_text=added_text, final=final) def parse_tool_call_complete(self, payload: str) -> ToolCall | None: return self._parse_tool_call_complete_json(payload) - - def _split(self, parsing_content: str): - """Split content into tuple: (text_content, tool_content, has_tool_end) - - This method parses the model output and separates it into regular text, - and tool call content. - """ - try: - start_idx = parsing_content.index(self.tool_start_token) - self.parse_cursor += start_idx - except ValueError: - # No new in this slice. - self.parse_cursor += len(parsing_content) - return parsing_content, '', False - try: - end_idx = parsing_content.index(self.tool_end_token) - except ValueError: - # Saw a start tag but not an end tag: enter tool block. - self.in_tool_block = True - return parsing_content[:start_idx], '', False - # Completed a full ... block in this slice. - self.parse_cursor += (end_idx - start_idx) + len(self.tool_end_token) - self.in_tool_block = False - return ( - parsing_content[:start_idx], - parsing_content[start_idx + len(self.tool_start_token):end_idx], - True, - ) diff --git a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py index 94207e1c22..35f7771a51 100644 --- a/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/qwen3coder_tool_parser.py @@ -2,7 +2,6 @@ from __future__ import annotations import json -import re from typing import TYPE_CHECKING, Any from lmdeploy.serve.openai.protocol import ( @@ -11,18 +10,15 @@ FunctionCall, ToolCall, ) -from lmdeploy.utils import get_logger from .tool_parser import ToolParser, ToolParserManager if TYPE_CHECKING: from lmdeploy.serve.openai.protocol import ChatCompletionRequest -logger = get_logger('lmdeploy') - def _parse_tool_call_arguments_dict(arguments: Any) -> dict[str, Any] | None: - """Return dict-like tool arguments for Qwen3Coder request rendering.""" + """Return dict-like tool arguments for Qwen3Coder request normalization.""" if not isinstance(arguments, str): return None @@ -37,12 +33,7 @@ def _parse_tool_call_arguments_dict(arguments: Any) -> dict[str, Any] | None: @ToolParserManager.register_module(['qwen3coder']) class Qwen3CoderToolParser(ToolParser): - """Parser for Qwen3 Coder model's tool call format. - - Handles the extraction of tool calls from Qwen3 Coder's output format, which uses purely XML tags for function names - and parameters, e.g., arg_value - - """ + """Tool parser for Qwen3Coder XML tool-call payloads.""" def __init__(self, tokenizer: object): super().__init__(tokenizer) @@ -52,11 +43,6 @@ def __init__(self, tokenizer: object): self.func_end_token = '' self.param_prefix = '(.*?)', re.DOTALL) - self.parse_cursor = 0 - self.qwen_tool_serial_index = -1 - self.qwen_active_tool_call_id = '' self.coder_has_emitted_name = False self.coder_has_emitted_json_start = False self.coder_json_closed = False @@ -126,8 +112,7 @@ def finish_tool_call(self) -> None: self.coder_emitted_param_names.clear() def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: - """Decode XML tool payload incrementally into OpenAI tool-call - deltas.""" + """Decode incremental XML tool payload.""" self._tool_payload += added_text func_name, args_dict, is_func_closed = self._extract_params(self._tool_payload) @@ -185,26 +170,8 @@ def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionReques return request return request.model_copy(update={'messages': normalized_messages}) - def _split(self, parsing_content: str) -> tuple[str, str, bool]: - """Split content into tuple: (text_content, tool_content, has_tool_end)""" - try: - start_idx = parsing_content.index(self.tool_start_token) - self.parse_cursor += start_idx - except ValueError: - self.parse_cursor += len(parsing_content) - return parsing_content, '', False - - try: - end_idx = parsing_content.index(self.tool_end_token) - except ValueError: - return parsing_content[:start_idx], parsing_content[start_idx:], False - - rem = end_idx - start_idx - self.parse_cursor += rem + len(self.tool_end_token) - return parsing_content[:start_idx], parsing_content[start_idx:end_idx + len(self.tool_end_token)], True - def _extract_params(self, content: str) -> tuple[str | None, dict[str, Any], bool]: - """Parse XML tool content into components.""" + """Extract function name, parameter map, and close status from XML.""" content = content.replace(self.tool_start_token, '').replace(self.tool_end_token, '').strip() func_name = None diff --git a/lmdeploy/serve/openai/tool_parser/tool_parser.py b/lmdeploy/serve/openai/tool_parser/tool_parser.py index 6d6f5f800e..69c65a99dc 100644 --- a/lmdeploy/serve/openai/tool_parser/tool_parser.py +++ b/lmdeploy/serve/openai/tool_parser/tool_parser.py @@ -27,10 +27,7 @@ class ToolParser: - """Abstract ToolParser class that should not be used directly. - - Provided properties and methods should be used in derived classes. - """ + """Base class for model-specific tool parsers.""" def __init__(self, tokenizer: object): self.model_tokenizer = tokenizer @@ -38,11 +35,7 @@ def __init__(self, tokenizer: object): self._active_tool_call_id: str = '' self._active_tool_index: int = -1 self._name_emitted: bool = False - self._args_prefix_emitted: bool = False - self._value_chars_emitted: int = 0 - self._args_closed_emitted: bool = False self._args_emitted_len: int = 0 - self._prev_args_json: str | None = None @cached_property def vocab(self) -> dict[str, int]: @@ -51,7 +44,7 @@ def vocab(self) -> dict[str, int]: return self.model_tokenizer.get_vocab() def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: - """Static method that used to adjust the request parameters.""" + """Adjust request payload before rendering, if needed.""" if request.tools is not None and request.tool_choice != 'none': if not isinstance(request.tool_choice, str): request.tools = [ @@ -79,22 +72,14 @@ def start_tool_call(self) -> None: self._active_tool_index += 1 self._active_tool_call_id = f'chatcmpl-tool-{shortuuid.random()}' self._name_emitted = False - self._args_prefix_emitted = False - self._value_chars_emitted = 0 - self._args_closed_emitted = False self._args_emitted_len = 0 - self._prev_args_json = None self._tool_payload = '' def finish_tool_call(self) -> None: """Mark end of a tool-call block.""" self._active_tool_call_id = '' self._name_emitted = False - self._args_prefix_emitted = False - self._value_chars_emitted = 0 - self._args_closed_emitted = False self._args_emitted_len = 0 - self._prev_args_json = None self._tool_payload = '' def decode_tool_incremental(self, added_text: str, *, final: bool) -> list[DeltaToolCall]: @@ -155,22 +140,6 @@ def _decode_tool_incremental_json(self, added_text: str, *, final: bool) -> list self._args_emitted_len = len(args_json) return out - @staticmethod - def _is_complete_json(text: str) -> bool: - try: - json.loads(text) - return True - except json.JSONDecodeError: - return False - - @staticmethod - def _common_prefix(s1: str, s2: str) -> str: - i = 0 - n = min(len(s1), len(s2)) - while i < n and s1[i] == s2[i]: - i += 1 - return s1[:i] - @staticmethod def _parse_tool_call_complete_json(payload: str) -> ToolCall | None: if not payload: diff --git a/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py b/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py index 7cf921ae6d..62156623bd 100644 --- a/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py +++ b/tests/test_lmdeploy/server/parsers/test_qwen3_5_parsers.py @@ -162,10 +162,29 @@ def test_stream_chunk_matches_reference(self, tokenizer, response_parser): delta_token_ids=delta_ids, ) + if delta_msg is None: + assert exp_reasoning is None + assert exp_content is None + assert exp_tool_emitted is False + assert tool_emitted is False + continue + assert delta_msg.reasoning_content == exp_reasoning if exp_content is not None: assert delta_msg.content == exp_content + # Tool-call expectations in this fixture are placeholders for now. + # Only enforce the exact tool_emitted flag when an explicit tool + # delta shape is provided. + if ( + exp_function_name is None + and exp_function_arguments is None + and exp_type is None + and exp_reasoning is None + and exp_content is None + ): + continue + assert tool_emitted == exp_tool_emitted if tool_emitted: From d02811842bfb9a6d7e1357dfc7ffb5172085cd5d Mon Sep 17 00:00:00 2001 From: lvhan028 Date: Thu, 2 Apr 2026 07:00:59 +0000 Subject: [PATCH 14/14] fix --- docs/en/llm/api_server_reasoning.md | 87 ++++++++---------- docs/zh_cn/llm/api_server_reasoning.md | 89 ++++++++----------- lmdeploy/cli/utils.py | 4 +- lmdeploy/serve/openai/api_server.py | 24 +---- .../reasoning_parser/reasoning_parser.py | 4 +- lmdeploy/serve/openai/response_parser.py | 30 +++++++ 6 files changed, 112 insertions(+), 126 deletions(-) diff --git a/docs/en/llm/api_server_reasoning.md b/docs/en/llm/api_server_reasoning.md index 88c475c480..67b73f5789 100644 --- a/docs/en/llm/api_server_reasoning.md +++ b/docs/en/llm/api_server_reasoning.md @@ -1,12 +1,12 @@ # Reasoning Outputs -For models that support reasoning capabilities, such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1), LMDeploy supports parsing the reasoning results in the service and separately records the reasoning content using `reasoning_content`. +For models that support reasoning capabilities, such as [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1), LMDeploy can parse reasoning outputs on the server side and expose them via `reasoning_content`. ## Examples ### DeepSeek R1 -We can start the DeepSeek R1 model's api_server service just like launching other models. The difference is that we need to specify --reasoning-parser\` parameter. +We can start DeepSeek R1's `api_server` like other models, but we need to specify the `--reasoning-parser` argument. ``` lmdeploy serve api_server deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --reasoning-parser deepseek-r1 @@ -44,62 +44,49 @@ print("content:", content) ## Custom parser -You only need to add a similar parser class in `lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py`. +Built-in reasoning parser names include: -```python -# import the required packages -from typing import Sequence, Union, Tuple, Optional +- `qwen-qwq` +- `qwen3` +- `intern-s1` +- `deepseek-r1` +- `deepseek-v3` +- `gpt-oss` + +### Notes + +- `deepseek-v3`: starts in reasoning mode only when `enable_thinking=True`. + When `enable_thinking` is `None` (default), output is usually plain content without a reasoning segment. +- `gpt-oss`: parses OpenAI Harmony channels: + - `final` -> `content` + - `analysis` -> `reasoning_content` + - `commentary` with `functions.*` recipient -> `tool_calls` + +### Add a custom parser + +Add a parser class under `lmdeploy/serve/openai/reasoning_parser/` and register it with `ReasoningParserManager`. +```python from lmdeploy.serve.openai.reasoning_parser import ( - ReasoningParser, ReasoningParserManager) -from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, - DeltaMessage) + ReasoningParser, ReasoningParserManager +) -# define a reasoning parser and register it to lmdeploy -# the name list in register_module can be used -# in --reasoning-parser. @ReasoningParserManager.register_module(["example"]) class ExampleParser(ReasoningParser): - def __init__(self, tokenizer: object): - super().__init__(tokenizer) - - def extract_reasoning_content_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> Union[DeltaMessage, None]: - """ - Instance method that should be implemented for extracting reasoning - from an incomplete response; for use when handling reasoning calls and - streaming. Has to be an instance method because it requires state - - the current tokens/diffs, but also the information about what has - previously been parsed and extracted (see constructor) - """ - - def extract_reasoning_content( - self, model_output: str, request: ChatCompletionRequest - ) -> Tuple[Optional[str], Optional[str]]: - """ - Extract reasoning content from a complete model-generated string. - - Used for non-streaming responses where we have the entire model response - available before sending to the client. - - Args: - model_output (str): The model-generated string to extract reasoning content from. - request (ChatCompletionRequest): he request object that was used to generate the model_output. - - Returns: - reasoning_content (str | None): The reasoning content. - final_output (str | None): The content. - """ + def __init__(self, tokenizer: object, **kwargs): + super().__init__(tokenizer, **kwargs) + + def get_reasoning_open_tag(self) -> str | None: + return "" + + def get_reasoning_close_tag(self) -> str | None: + return "" + + def starts_in_reasoning_mode(self) -> bool: + return True ``` -Similarly, the command to start the service becomes: +Then start the service with: ``` lmdeploy serve api_server $model_path --reasoning-parser example diff --git a/docs/zh_cn/llm/api_server_reasoning.md b/docs/zh_cn/llm/api_server_reasoning.md index 4860cd1553..9cf54941ce 100644 --- a/docs/zh_cn/llm/api_server_reasoning.md +++ b/docs/zh_cn/llm/api_server_reasoning.md @@ -1,14 +1,12 @@ # Reasoning Outputs -对于支持推理能力的模型，比如 [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1)，LMDeploy 支持在服务中将推理的结果解析出来，并单独用 -reasoning_content 记录推理内容。 +对于支持推理能力的模型，比如 [DeepSeek R1](https://huggingface.co/deepseek-ai/DeepSeek-R1)，LMDeploy 支持在服务端解析推理结果，并通过 `reasoning_content` 单独返回推理内容。 ## 使用示例 ### DeepSeek R1 -我们可以像启动其他模型的 api_server 服务一样启动 DeepSeek R1 的模型，只是不同的是，我们需要指定 `--reasoning-parser`。 -在 `--reasoning-parser` 传参里，我们需要指定具体的 parser。 +我们可以像启动其他模型一样启动 DeepSeek R1 的 `api_server`，但需要额外指定 `--reasoning-parser` 参数。 ``` lmdeploy serve api_server deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B --reasoning-parser deepseek-r1 @@ -46,62 +44,49 @@ print("content:", content) ## 自定义 parser -只需要在 `lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py` 中添加一个类似的 parser 类即可。 +内置的 reasoning parser 名称包括： -```python -# import the required packages -from typing import Sequence, Union, Tuple, Optional +- `qwen-qwq` +- `qwen3` +- `intern-s1` +- `deepseek-r1` +- `deepseek-v3` +- `gpt-oss` + +### 说明 + +- `deepseek-v3`：仅当 `enable_thinking=True` 时，才会从推理模式开始解析。 + 当 `enable_thinking` 为 `None`（默认）时，通常不会出现推理段，输出为普通内容。 +- `gpt-oss`：基于 OpenAI Harmony channel 解析： + - `final` -> `content` + - `analysis` -> `reasoning_content` + - `commentary` 且 `recipient` 为 `functions.*` -> `tool_calls` + +### 添加自定义 parser + +在 `lmdeploy/serve/openai/reasoning_parser/` 目录下新增 parser 类，并通过 `ReasoningParserManager` 注册。 +```python from lmdeploy.serve.openai.reasoning_parser import ( - ReasoningParser, ReasoningParserManager) -from lmdeploy.serve.openai.protocol import (ChatCompletionRequest, - DeltaMessage) + ReasoningParser, ReasoningParserManager +) -# define a reasoning parser and register it to lmdeploy -# the name list in register_module can be used -# in --reasoning-parser. @ReasoningParserManager.register_module(["example"]) class ExampleParser(ReasoningParser): - def __init__(self, tokenizer: object): - super().__init__(tokenizer) - - def extract_reasoning_content_streaming( - self, - previous_text: str, - current_text: str, - delta_text: str, - previous_token_ids: Sequence[int], - current_token_ids: Sequence[int], - delta_token_ids: Sequence[int], - ) -> Union[DeltaMessage, None]: - """ - Instance method that should be implemented for extracting reasoning - from an incomplete response; for use when handling reasoning calls and - streaming. Has to be an instance method because it requires state - - the current tokens/diffs, but also the information about what has - previously been parsed and extracted (see constructor) - """ - - def extract_reasoning_content( - self, model_output: str, request: ChatCompletionRequest - ) -> Tuple[Optional[str], Optional[str]]: - """ - Extract reasoning content from a complete model-generated string. - - Used for non-streaming responses where we have the entire model response - available before sending to the client. - - Args: - model_output (str): The model-generated string to extract reasoning content from. - request (ChatCompletionRequest): he request object that was used to generate the model_output. - - Returns: - reasoning_content (str | None): The reasoning content. - final_output (str | None): The content. - """ + def __init__(self, tokenizer: object, **kwargs): + super().__init__(tokenizer, **kwargs) + + def get_reasoning_open_tag(self) -> str | None: + return "" + + def get_reasoning_close_tag(self) -> str | None: + return "" + + def starts_in_reasoning_mode(self) -> bool: + return True ``` -类似的，启动服务的命令就变成了： +然后通过以下命令启动服务： ``` lmdeploy serve api_server $model_path --reasoning-parser example diff --git a/lmdeploy/cli/utils.py b/lmdeploy/cli/utils.py index 70dea1a535..9f808dd411 100644 --- a/lmdeploy/cli/utils.py +++ b/lmdeploy/cli/utils.py @@ -462,12 +462,14 @@ def chat_template(parser): @staticmethod def reasoning_parser(parser): """Add reasoning parser to parser.""" + legacy_names = ['qwen-qwq', 'intern-s1', 'deepseek-r1'] from lmdeploy.serve.openai.reasoning_parser import ReasoningParserManager return parser.add_argument( '--reasoning-parser', type=str, default=None, - help=f'The registered reasoning parser name from {ReasoningParserManager.module_dict.keys()}. ' + help=f'The registered reasoning parser name: {ReasoningParserManager.module_dict.keys()}. ' + f'Legacy names: {legacy_names}. ' 'Default to None.') @staticmethod diff --git a/lmdeploy/serve/openai/api_server.py b/lmdeploy/serve/openai/api_server.py index 7a5c19e10e..377ab4c3bd 100644 --- a/lmdeploy/serve/openai/api_server.py +++ b/lmdeploy/serve/openai/api_server.py @@ -75,9 +75,7 @@ UpdateParamsRequest, UsageInfo, ) -from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager from lmdeploy.serve.openai.response_parser import ResponseParser -from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParserManager from lmdeploy.serve.utils.server_utils import validate_json_request from lmdeploy.utils import get_logger @@ -470,7 +468,6 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: completion_tokens=res.generate_token_len, total_tokens=total_tokens, ) - print(f'[completion_stream_generator] res.response: {res.response}, res.token_ids: {res.token_ids}') delta_token_ids = res.token_ids if res.token_ids is not None else [] delta_message, tool_emitted = response_parser.stream_chunk( res.response, @@ -557,8 +554,7 @@ async def completion_stream_generator() -> AsyncGenerator[str, None]: logprobs = None if gen_logprobs and len(final_logprobs): - logprobs = _create_chat_completion_logprobs(VariableInterface.async_engine.tokenizer, final_token_ids, - final_logprobs) + logprobs = _create_chat_completion_logprobs(tokenizer, final_token_ids, final_logprobs) assert final_res is not None choices = [] @@ -1200,19 +1196,7 @@ async def dispatch(self, request: Request, call_next): def set_parsers(reasoning_parser_name: str | None = None, tool_parser_name: str | None = None, **kwargs): """Set tool parser and reasoning parser types on :class:`~lmdeploy.serve.openai.response_parser.ResponseParser`.""" - if reasoning_parser_name is not None: - if reasoning_parser_name in ReasoningParserManager.module_dict: - ResponseParser.reasoning_parser_cls = ReasoningParserManager.get(reasoning_parser_name) - else: - raise ValueError(f'The reasoning parser {reasoning_parser_name} is not in the parser list: ' - f'{ReasoningParserManager.module_dict.keys()}') - - if tool_parser_name is not None: - if tool_parser_name in ToolParserManager.module_dict: - ResponseParser.tool_parser_cls = ToolParserManager.get(tool_parser_name) - else: - raise ValueError(f'The tool parser {tool_parser_name} is not in the parser list: ' - f'{ToolParserManager.module_dict.keys()}') + ResponseParser.set_parsers(reasoning_parser_name=reasoning_parser_name, tool_parser_name=tool_parser_name) def mount_metrics(app: FastAPI, backend_config: PytorchEngineConfig | TurbomindEngineConfig): @@ -1351,6 +1335,8 @@ def serve(model_path: str, ssl_certfile = os.environ['SSL_CERTFILE'] http_or_https = 'https' + set_parsers(reasoning_parser, tool_call_parser) + handle_torchrun() _, pipeline_class = get_task(backend, model_path) if isinstance(backend_config, PytorchEngineConfig): @@ -1366,8 +1352,6 @@ def serve(model_path: str, max_log_len=max_log_len, speculative_config=speculative_config, **kwargs) - # set reasoning parser and tool parser - set_parsers(reasoning_parser, tool_call_parser) # create FastAPI lifespan events lifespan = create_lifespan_handler(backend_config, VariableInterface.async_engine) diff --git a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py index d4165da920..f9b1ac5d43 100644 --- a/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py +++ b/lmdeploy/serve/openai/reasoning_parser/reasoning_parser.py @@ -11,9 +11,7 @@ ReasoningParserManager = Registry('reasoning_parser', locations=['lmdeploy.serve.openai.reasoning_parser']) -@ReasoningParserManager.register_module(name=[ - 'qwen-qwq', 'qwen3', 'intern-s1', 'deepseek-r1' -]) +@ReasoningParserManager.register_module(name='default') class ReasoningParser: """Unified reasoning parser for all ``--reasoning-parser`` options.""" diff --git a/lmdeploy/serve/openai/response_parser.py b/lmdeploy/serve/openai/response_parser.py index 5468cc1174..b9dded75ab 100644 --- a/lmdeploy/serve/openai/response_parser.py +++ b/lmdeploy/serve/openai/response_parser.py @@ -67,6 +67,36 @@ class ResponseParser: MODE_REASONING: ClassVar[str] = 'reasoning' MODE_TOOL: ClassVar[str] = 'tool' + @classmethod + def set_parsers( + cls, + reasoning_parser_name: str | None = None, + tool_parser_name: str | None = None, + ) -> None: + """Configure reasoning/tool parser classes by registry name.""" + from lmdeploy.serve.openai.reasoning_parser.reasoning_parser import ReasoningParserManager + from lmdeploy.serve.openai.tool_parser.tool_parser import ToolParserManager + + legacy_reasoning_parser_names = ['qwen-qwq', 'intern-s1', 'deepseek-r1'] + if reasoning_parser_name in legacy_reasoning_parser_names: + logger.warning(f'The reasoning parser {reasoning_parser_name} is deprecated, ' + 'please use the default reasoning parser instead.') + reasoning_parser_name = 'default' + + if reasoning_parser_name is not None: + if reasoning_parser_name in ReasoningParserManager.module_dict: + cls.reasoning_parser_cls = ReasoningParserManager.get(reasoning_parser_name) + else: + raise ValueError(f'The reasoning parser {reasoning_parser_name} is not in the parser list: ' + f'{ReasoningParserManager.module_dict.keys()}') + + if tool_parser_name is not None: + if tool_parser_name in ToolParserManager.module_dict: + cls.tool_parser_cls = ToolParserManager.get(tool_parser_name) + else: + raise ValueError(f'The tool parser {tool_parser_name} is not in the parser list: ' + f'{ToolParserManager.module_dict.keys()}') + @classmethod def chat_template_kwargs_from_request(cls, request: ChatCompletionRequest) -> dict: """Normalize parser-related template kwargs from the request.