From 3f64f5faf5ef89dabbc6436e8ca75ec2f4409148 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 10:28:45 -0400 Subject: [PATCH 01/10] x --- .../langchain_v1/langchain/tools/tool_node.py | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/libs/langchain_v1/langchain/tools/tool_node.py b/libs/langchain_v1/langchain/tools/tool_node.py index 54843340317a3..67bc321c0954b 100644 --- a/libs/langchain_v1/langchain/tools/tool_node.py +++ b/libs/langchain_v1/langchain/tools/tool_node.py @@ -623,16 +623,11 @@ def _func( ) tool_runtimes.append(tool_runtime) - # Inject tool arguments (including runtime) - - injected_tool_calls = [] + # Pass original tool calls without injection input_types = [input_type] * len(tool_calls) - for call, tool_runtime in zip(tool_calls, tool_runtimes, strict=False): - injected_call = self._inject_tool_args(call, tool_runtime) # type: ignore[arg-type] - injected_tool_calls.append(injected_call) with get_executor_for_config(config) as executor: outputs = list( - executor.map(self._run_one, injected_tool_calls, input_types, tool_runtimes) + executor.map(self._run_one, tool_calls, input_types, tool_runtimes) ) return self._combine_tool_outputs(outputs, input_type) @@ -660,12 +655,10 @@ async def _afunc( ) tool_runtimes.append(tool_runtime) - injected_tool_calls = [] + # Pass original tool calls without injection coros = [] for call, tool_runtime in zip(tool_calls, tool_runtimes, strict=False): - injected_call = self._inject_tool_args(call, tool_runtime) # type: ignore[arg-type] - injected_tool_calls.append(injected_call) - coros.append(self._arun_one(injected_call, input_type, tool_runtime)) # type: ignore[arg-type] + coros.append(self._arun_one(call, input_type, tool_runtime)) # type: ignore[arg-type] outputs = await asyncio.gather(*coros) return self._combine_tool_outputs(outputs, input_type) @@ -742,12 +735,15 @@ def _execute_tool_sync( msg = f"Tool {call['name']} is not registered with ToolNode" raise TypeError(msg) - call_args = {**call, "type": "tool_call"} + # Inject state, store, and runtime right before invocation + injected_call = self._inject_tool_args(call, request.runtime) + call_args = {**injected_call, "type": "tool_call"} try: try: response = tool.invoke(call_args, config) except ValidationError as exc: + # Use original call["args"] without injected values for error reporting raise ToolInvocationError(call["name"], exc, call["args"]) from exc # GraphInterrupt is a special exception that will always be raised. @@ -887,12 +883,15 @@ async def _execute_tool_async( msg = f"Tool {call['name']} is not registered with ToolNode" raise TypeError(msg) - call_args = {**call, "type": "tool_call"} + # Inject state, store, and runtime right before invocation + injected_call = self._inject_tool_args(call, request.runtime) + call_args = {**injected_call, "type": "tool_call"} try: try: response = await tool.ainvoke(call_args, config) except ValidationError as exc: + # Use original call["args"] without injected values for error reporting raise ToolInvocationError(call["name"], exc, call["args"]) from exc # GraphInterrupt is a special exception that will always be raised. From 35081b1eac89ea77cc15cea4e324ccab25d7fcb5 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 10:58:05 -0400 Subject: [PATCH 02/10] x --- .../tests/unit_tests/agents/test_tool_node.py | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py index 0f04f98832b28..24326e88ea8c1 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py @@ -10,6 +10,8 @@ TypeVar, ) from unittest.mock import Mock +from langchain.agents import create_agent +from langchain.agents.middleware.types import AgentState import pytest from langchain_core.messages import ( @@ -302,6 +304,70 @@ def test_tool_node_error_handling_default_exception() -> None: ) +def test_tool_invocation_error_excludes_injected_state() -> None: + """Test that tool invocation errors do not include injected state information. + + When a tool has InjectedState parameters and the LLM makes an incorrect + invocation (e.g., missing required arguments), the error message should only + contain the original arguments from the tool call, not the injected state values. + + This test uses create_agent to ensure the behavior works in a full agent context. + """ + # Define a custom state schema with secret data + class TestState(AgentState): + secret_data: str + + @dec_tool + def tool_with_injected_state( + some_val: int, + state: Annotated[TestState, InjectedState], + ) -> str: + """Tool that uses injected state.""" + return f"some_val: {some_val}" + + # Create a fake model that makes an incorrect tool call (missing 'some_val') + # Then returns no tool calls on the second iteration to end the loop + model = FakeToolCallingModel( + tool_calls=[ + [ + { + "name": "tool_with_injected_state", + "args": {"wrong_arg": "value"}, # Missing required 'some_val' + "id": "call_1", + } + ], + [], # No tool calls on second iteration to end the loop + ] + ) + + # Create an agent with the tool and custom state schema + agent = create_agent( + model=model, + tools=[tool_with_injected_state], + state_schema=TestState, + ) + + # Invoke the agent with secret data in the state + result = agent.invoke( + { + "messages": [HumanMessage("Test message")], + "secret_data": "sensitive_secret_123", + } + ) + + # Find the tool error message + tool_messages = [m for m in result["messages"] if m.type == "tool"] + assert len(tool_messages) == 1 + tool_message = tool_messages[0] + assert tool_message is None + assert tool_message.status == "error" + + # The error message should contain the original args in the kwargs section + assert "{'wrong_arg': 'value'}" in tool_message.content + assert "secret_data" not in tool_message.content + assert "sensitive_secret_123" not in tool_message.content + + async def test_tool_node_error_handling() -> None: def handle_all(e: ValueError | ToolException | ToolInvocationError): return TOOL_CALL_ERROR_TEMPLATE.format(error=repr(e)) From e0372c14c06673ef4f66666cd482c51a72a0ccb0 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 11:42:22 -0400 Subject: [PATCH 03/10] x --- .../langchain_v1/langchain/tools/tool_node.py | 97 +++++++++++++++++-- .../tests/unit_tests/agents/test_tool_node.py | 2 +- 2 files changed, 91 insertions(+), 8 deletions(-) diff --git a/libs/langchain_v1/langchain/tools/tool_node.py b/libs/langchain_v1/langchain/tools/tool_node.py index 67bc321c0954b..60c5ee47b0c70 100644 --- a/libs/langchain_v1/langchain/tools/tool_node.py +++ b/libs/langchain_v1/langchain/tools/tool_node.py @@ -89,6 +89,7 @@ def my_tool(x: int) -> str: from collections.abc import Sequence from langgraph.runtime import Runtime + from pydantic_core import ErrorDetails # right now we use a dict as the default, can change this to AgentState, but depends # on if this lives in LangChain or LangGraph... ideally would have some typed @@ -303,7 +304,11 @@ class ToolInvocationError(ToolException): """ def __init__( - self, tool_name: str, source: ValidationError, tool_kwargs: dict[str, Any] + self, + tool_name: str, + source: ValidationError, + tool_kwargs: dict[str, Any], + filtered_errors: list[ErrorDetails] | None = None, ) -> None: """Initialize the ToolInvocationError. @@ -311,13 +316,31 @@ def __init__( tool_name: The name of the tool that failed. source: The exception that occurred. tool_kwargs: The keyword arguments that were passed to the tool. + filtered_errors: Optional list of filtered validation errors excluding + injected arguments. """ + # Format error display based on filtered errors if provided + if filtered_errors is not None: + # Manually format the filtered errors + error_count = len(filtered_errors) + plural = "s" if error_count != 1 else "" + header = f"{error_count} validation error{plural} for {tool_name}" + error_str_parts = [header] + for error in filtered_errors: + loc_str = " -> ".join(str(loc) for loc in error.get("loc", ())) + error_str_parts.append(f"{loc_str}") + error_str_parts.append(f" {error.get('msg', 'Unknown error')}") + error_display_str = "\n".join(error_str_parts) + else: + error_display_str = str(source) + self.message = TOOL_INVOCATION_ERROR_TEMPLATE.format( - tool_name=tool_name, tool_kwargs=tool_kwargs, error=source + tool_name=tool_name, tool_kwargs=tool_kwargs, error=error_display_str ) self.tool_name = tool_name self.tool_kwargs = tool_kwargs self.source = source + self.filtered_errors = filtered_errors super().__init__(self.message) @@ -442,6 +465,54 @@ def _infer_handled_types(handler: Callable[..., str]) -> tuple[type[Exception], return (Exception,) +def _filter_validation_errors( + validation_error: ValidationError, + tool_to_state_args: dict[str, str | None], + tool_to_store_arg: str | None, + tool_to_runtime_arg: str | None, +) -> list[ErrorDetails]: + """Filter validation errors to exclude injected arguments. + + When a tool is invoked with arguments that fail validation, we need to filter + out any errors associated with injected arguments (state, store, runtime) since + these are not provided by the model and should not be reported as invocation errors. + + Args: + validation_error: The Pydantic ValidationError raised during tool invocation. + tool_to_state_args: Mapping of state argument names to state field names. + tool_to_store_arg: Name of the store argument, if any. + tool_to_runtime_arg: Name of the runtime argument, if any. + + Returns: + List of ErrorDetails with injected argument errors filtered out and + injected values removed from input_value fields. + """ + injected_args = set(tool_to_state_args.keys()) + if tool_to_store_arg: + injected_args.add(tool_to_store_arg) + if tool_to_runtime_arg: + injected_args.add(tool_to_runtime_arg) + + filtered_errors: list[ErrorDetails] = [] + for error in validation_error.errors(): + # Check if error location contains any injected argument + # error['loc'] is a tuple like ('field_name',) or ('field_name', 'nested_field') + if error["loc"] and error["loc"][0] not in injected_args: + # Create a copy of the error dict to avoid mutating the original + error_copy: dict[str, Any] = {**error} + + # Remove injected arguments from input_value if it's a dict + if isinstance(error_copy.get("input"), dict): + input_dict = error_copy["input"] + input_copy = {k: v for k, v in input_dict.items() if k not in injected_args} + error_copy["input"] = input_copy + + # Cast is safe because ErrorDetails is a TypedDict compatible with this structure + filtered_errors.append(error_copy) # type: ignore[arg-type] + + return filtered_errors + + class _ToolNode(RunnableCallable): """A node for executing tools in LangGraph workflows. @@ -626,9 +697,7 @@ def _func( # Pass original tool calls without injection input_types = [input_type] * len(tool_calls) with get_executor_for_config(config) as executor: - outputs = list( - executor.map(self._run_one, tool_calls, input_types, tool_runtimes) - ) + outputs = list(executor.map(self._run_one, tool_calls, input_types, tool_runtimes)) return self._combine_tool_outputs(outputs, input_type) @@ -743,8 +812,15 @@ def _execute_tool_sync( try: response = tool.invoke(call_args, config) except ValidationError as exc: + # Filter out errors for injected arguments + filtered_errors = _filter_validation_errors( + exc, + self._tool_to_state_args.get(call["name"], {}), + self._tool_to_store_arg.get(call["name"]), + self._tool_to_runtime_arg.get(call["name"]), + ) # Use original call["args"] without injected values for error reporting - raise ToolInvocationError(call["name"], exc, call["args"]) from exc + raise ToolInvocationError(call["name"], exc, call["args"], filtered_errors) from exc # GraphInterrupt is a special exception that will always be raised. # It can be triggered in the following scenarios, @@ -891,8 +967,15 @@ async def _execute_tool_async( try: response = await tool.ainvoke(call_args, config) except ValidationError as exc: + # Filter out errors for injected arguments + filtered_errors = _filter_validation_errors( + exc, + self._tool_to_state_args.get(call["name"], {}), + self._tool_to_store_arg.get(call["name"]), + self._tool_to_runtime_arg.get(call["name"]), + ) # Use original call["args"] without injected values for error reporting - raise ToolInvocationError(call["name"], exc, call["args"]) from exc + raise ToolInvocationError(call["name"], exc, call["args"], filtered_errors) from exc # GraphInterrupt is a special exception that will always be raised. # It can be triggered in the following scenarios, diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py index 24326e88ea8c1..7168916a786fb 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py @@ -313,6 +313,7 @@ def test_tool_invocation_error_excludes_injected_state() -> None: This test uses create_agent to ensure the behavior works in a full agent context. """ + # Define a custom state schema with secret data class TestState(AgentState): secret_data: str @@ -359,7 +360,6 @@ def tool_with_injected_state( tool_messages = [m for m in result["messages"] if m.type == "tool"] assert len(tool_messages) == 1 tool_message = tool_messages[0] - assert tool_message is None assert tool_message.status == "error" # The error message should contain the original args in the kwargs section From cbd75b04d83b6fdd696d4934a33affc2f817d03b Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 11:42:33 -0400 Subject: [PATCH 04/10] x --- ...st_tool_node_validation_error_filtering.py | 629 ++++++++++++++++++ 1 file changed, 629 insertions(+) create mode 100644 libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py new file mode 100644 index 0000000000000..67d497b092821 --- /dev/null +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py @@ -0,0 +1,629 @@ +"""Unit tests for ValidationError filtering in ToolNode. + +This module tests that validation errors for injected arguments (InjectedState, +InjectedStore, ToolRuntime) are properly filtered out when tools are invoked with +invalid arguments, ensuring only model-controllable argument errors are reported. +""" + +from typing import Annotated +from unittest.mock import Mock + +import pytest +from langchain_core.messages import AIMessage, HumanMessage +from langchain_core.runnables.config import RunnableConfig +from langchain_core.tools import tool as dec_tool +from langgraph.store.base import BaseStore +from langgraph.store.memory import InMemoryStore +from pydantic import BaseModel +from typing_extensions import TypedDict + +from langchain.agents import create_agent +from langchain.agents.middleware.types import AgentState +from langchain.tools import InjectedState, InjectedStore +from langchain.tools.tool_node import ToolInvocationError, ToolRuntime, _ToolNode + +from .model import FakeToolCallingModel + +pytestmark = pytest.mark.anyio + + +def _create_mock_runtime(store: BaseStore | None = None) -> Mock: + """Create a mock Runtime object for testing ToolNode outside of graph context.""" + mock_runtime = Mock() + mock_runtime.store = store + mock_runtime.context = None + mock_runtime.stream_writer = lambda *args, **kwargs: None + return mock_runtime + + +def _create_config_with_runtime(store: BaseStore | None = None) -> RunnableConfig: + """Create a RunnableConfig with mock Runtime for testing ToolNode.""" + return {"configurable": {"__pregel_runtime": _create_mock_runtime(store)}} + + +async def test_filter_injected_state_validation_errors() -> None: + """Test that validation errors for InjectedState arguments are filtered out.""" + + @dec_tool + def my_tool( + value: int, + state: Annotated[dict, InjectedState], + ) -> str: + """Tool that uses injected state. + + Args: + value: An integer value. + state: The graph state (injected). + """ + return f"value={value}, messages={len(state.get('messages', []))}" + + tool_node = _ToolNode([my_tool]) + + # Call with invalid 'value' argument (should be int, not str) + result = await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {"value": "not_an_int"}, # Invalid type + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(), + ) + + # Should get a ToolMessage with error + assert len(result["messages"]) == 1 + tool_message = result["messages"][0] + assert tool_message.status == "error" + assert tool_message.tool_call_id == "call_1" + + # Error should mention 'value' but NOT 'state' (which is injected) + assert "value" in tool_message.content + assert "state" not in tool_message.content.lower() + + +async def test_filter_injected_store_validation_errors() -> None: + """Test that validation errors for InjectedStore arguments are filtered out.""" + + @dec_tool + def my_tool( + key: str, + store: Annotated[BaseStore, InjectedStore()], + ) -> str: + """Tool that uses injected store. + + Args: + key: A key to look up. + store: The persistent store (injected). + """ + return f"key={key}" + + tool_node = _ToolNode([my_tool]) + + # Call with invalid 'key' argument (missing required argument) + result = await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {}, # Missing 'key' + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(store=InMemoryStore()), + ) + + # Should get a ToolMessage with error + assert len(result["messages"]) == 1 + tool_message = result["messages"][0] + assert tool_message.status == "error" + + # Error should mention 'key' is required + assert "key" in tool_message.content.lower() + # The error should be about 'key' field specifically (not about store field) + # Note: 'store' might appear in input_value representation, but the validation + # error itself should only be for 'key' + assert ( + "field required" in tool_message.content.lower() + or "missing" in tool_message.content.lower() + ) + + +async def test_filter_tool_runtime_validation_errors() -> None: + """Test that validation errors for ToolRuntime arguments are filtered out.""" + + @dec_tool + def my_tool( + query: str, + runtime: ToolRuntime, + ) -> str: + """Tool that uses ToolRuntime. + + Args: + query: A query string. + runtime: The tool runtime context (injected). + """ + return f"query={query}" + + tool_node = _ToolNode([my_tool]) + + # Call with invalid 'query' argument (wrong type) + result = await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {"query": 123}, # Should be str, not int + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(), + ) + + # Should get a ToolMessage with error + assert len(result["messages"]) == 1 + tool_message = result["messages"][0] + assert tool_message.status == "error" + + # Error should mention 'query' but NOT 'runtime' (which is injected) + assert "query" in tool_message.content.lower() + assert "runtime" not in tool_message.content.lower() + + +async def test_filter_multiple_injected_args() -> None: + """Test filtering when multiple injected arguments have validation errors.""" + + @dec_tool + def my_tool( + value: int, + state: Annotated[dict, InjectedState], + store: Annotated[BaseStore, InjectedStore()], + runtime: ToolRuntime, + ) -> str: + """Tool with multiple injected arguments. + + Args: + value: An integer value. + state: The graph state (injected). + store: The persistent store (injected). + runtime: The tool runtime context (injected). + """ + return f"value={value}" + + tool_node = _ToolNode([my_tool]) + + # Call with invalid 'value' - injected args should be filtered from error + result = await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {"value": "not_an_int"}, + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(store=InMemoryStore()), + ) + + tool_message = result["messages"][0] + assert tool_message.status == "error" + + # Only 'value' error should be reported + assert "value" in tool_message.content + # None of the injected args should appear in error + assert "state" not in tool_message.content.lower() + assert "store" not in tool_message.content.lower() + assert "runtime" not in tool_message.content.lower() + + +async def test_no_filtering_when_all_errors_are_model_args() -> None: + """Test that filtering doesn't hide errors for non-injected arguments.""" + + @dec_tool + def my_tool( + value1: int, + value2: str, + state: Annotated[dict, InjectedState], + ) -> str: + """Tool with both regular and injected arguments. + + Args: + value1: First value. + value2: Second value. + state: The graph state (injected). + """ + return f"value1={value1}, value2={value2}" + + tool_node = _ToolNode([my_tool]) + + # Call with invalid arguments for BOTH non-injected parameters + result = await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": { + "value1": "not_an_int", # Invalid + "value2": 456, # Invalid (should be str) + }, + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(), + ) + + tool_message = result["messages"][0] + assert tool_message.status == "error" + + # Both errors should be present + assert "value1" in tool_message.content + assert "value2" in tool_message.content + # Injected state should not appear + assert "state" not in tool_message.content.lower() + + +async def test_validation_error_with_no_injected_args() -> None: + """Test that normal tools without injection still show all errors.""" + + @dec_tool + def my_tool(value1: int, value2: str) -> str: + """Regular tool without injected arguments. + + Args: + value1: First value. + value2: Second value. + """ + return f"{value1} {value2}" + + tool_node = _ToolNode([my_tool]) + + result = await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {"value1": "invalid", "value2": 123}, + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(), + ) + + tool_message = result["messages"][0] + assert tool_message.status == "error" + + # Both errors should be present since there are no injected args to filter + assert "value1" in tool_message.content + assert "value2" in tool_message.content + + +async def test_tool_invocation_error_without_handle_errors() -> None: + """Test that ToolInvocationError is raised with filtered errors when not handling.""" + + @dec_tool + def my_tool( + value: int, + state: Annotated[dict, InjectedState], + ) -> str: + """Tool with injected state. + + Args: + value: An integer value. + state: The graph state (injected). + """ + return f"value={value}" + + tool_node = _ToolNode([my_tool], handle_tool_errors=False) + + # Should raise ToolInvocationError with filtered errors + with pytest.raises(ToolInvocationError) as exc_info: + await tool_node.ainvoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {"value": "not_an_int"}, + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(), + ) + + error = exc_info.value + assert error.tool_name == "my_tool" + assert error.filtered_errors is not None + assert len(error.filtered_errors) > 0 + + # Filtered errors should only contain 'value' error, not 'state' + error_locs = [err["loc"] for err in error.filtered_errors] + assert any("value" in str(loc) for loc in error_locs) + assert not any("state" in str(loc) for loc in error_locs) + + +async def test_sync_tool_validation_error_filtering() -> None: + """Test that error filtering works for sync tools as well.""" + + @dec_tool + def my_tool( + value: int, + state: Annotated[dict, InjectedState], + ) -> str: + """Sync tool with injected state. + + Args: + value: An integer value. + state: The graph state (injected). + """ + return f"value={value}" + + tool_node = _ToolNode([my_tool]) + + # Test sync invocation + result = tool_node.invoke( + { + "messages": [ + AIMessage( + "hi?", + tool_calls=[ + { + "name": "my_tool", + "args": {"value": "not_an_int"}, + "id": "call_1", + "type": "tool_call", + } + ], + ) + ] + }, + config=_create_config_with_runtime(), + ) + + tool_message = result["messages"][0] + assert tool_message.status == "error" + assert "value" in tool_message.content + assert "state" not in tool_message.content.lower() + + +async def test_create_agent_error_content_with_multiple_params() -> None: + """Test that error messages contain non-injected params but not injected ones. + + This test uses create_agent to verify that when a tool with both regular + and injected parameters receives invalid arguments, the error message: + 1. Contains details about the non-injected parameter errors + 2. Does NOT contain any injected parameter names or values + 3. Properly formats the validation errors for clarity + """ + + # Custom state with sensitive information + class TestState(AgentState): + user_id: str + api_key: str + session_data: dict + + @dec_tool + def complex_tool( + query: str, + limit: int, + state: Annotated[TestState, InjectedState], + store: Annotated[BaseStore, InjectedStore()], + runtime: ToolRuntime, + ) -> str: + """A complex tool with multiple injected and non-injected parameters. + + Args: + query: The search query string. + limit: Maximum number of results to return. + state: The graph state (injected). + store: The persistent store (injected). + runtime: The tool runtime context (injected). + """ + # Access injected params to verify they work in normal execution + user = state.get("user_id", "unknown") + return f"Results for '{query}' (limit={limit}, user={user})" + + # Create a model that makes an incorrect tool call with multiple errors: + # - query is wrong type (int instead of str) + # - limit is missing + # Then returns no tool calls to end the loop + model = FakeToolCallingModel( + tool_calls=[ + [ + { + "name": "complex_tool", + "args": { + "query": 12345, # Wrong type - should be str + # "limit" is missing - required field + }, + "id": "call_complex_1", + } + ], + [], # No tool calls on second iteration to end the loop + ] + ) + + # Create an agent with the complex tool and custom state + # Need to provide a store since the tool uses InjectedStore + agent = create_agent( + model=model, + tools=[complex_tool], + state_schema=TestState, + store=InMemoryStore(), + ) + + # Invoke with sensitive data in state + result = agent.invoke( + { + "messages": [HumanMessage("Search for something")], + "user_id": "user_12345", + "api_key": "sk-secret-key-abc123xyz", + "session_data": {"token": "secret_session_token"}, + } + ) + + # Find the tool error message + tool_messages = [m for m in result["messages"] if m.type == "tool"] + assert len(tool_messages) == 1 + tool_message = tool_messages[0] + assert tool_message.status == "error" + assert tool_message.tool_call_id == "call_complex_1" + + content = tool_message.content + + # Verify error mentions the non-injected parameter issues + # Should mention 'query' error + assert "query" in content.lower(), "Error should mention 'query' parameter" + + # Should mention 'limit' error (missing required field) + assert "limit" in content.lower(), "Error should mention 'limit' parameter" + + # Should indicate validation errors occurred + assert "validation error" in content.lower() or "error" in content.lower(), ( + "Error should indicate validation occurred" + ) + + # CRITICAL: Verify NO injected parameter names appear in error + assert "state" not in content.lower(), "Error should NOT mention 'state' (injected parameter)" + assert "store" not in content.lower(), "Error should NOT mention 'store' (injected parameter)" + assert "runtime" not in content.lower(), ( + "Error should NOT mention 'runtime' (injected parameter)" + ) + + # CRITICAL: Verify NO sensitive values from state appear in error + assert "user_12345" not in content, "Error should NOT contain user_id value from state" + assert "sk-secret-key" not in content, "Error should NOT contain api_key value from state" + assert "secret_session_token" not in content, "Error should NOT contain session data from state" + + # Verify the original tool call args are mentioned (not the injected ones) + # The error template includes: "with kwargs {tool_kwargs}" + # This should show the original args from the model's tool call + assert "12345" in content, "Error should show the invalid query value (12345)" + + # Additional verification: check error structure + # Should be formatted in a readable way + assert "complex_tool" in content, "Error should mention the tool name" + + +async def test_create_agent_error_only_model_controllable_params() -> None: + """Test that errors only show model-controllable parameter issues. + + This is a focused test ensuring that when ONLY non-injected parameters + have validation errors, those errors are clearly shown without any + confusion from injected parameters. + """ + + class StateWithSecrets(AgentState): + password: str + + @dec_tool + def secure_tool( + username: str, + email: str, + state: Annotated[StateWithSecrets, InjectedState], + ) -> str: + """Tool that validates user credentials. + + Args: + username: The username (3-20 chars). + email: The email address. + state: State with password (injected). + """ + return f"Validated {username} with email {email}" + + # Model provides invalid username (too short) and invalid email + model = FakeToolCallingModel( + tool_calls=[ + [ + { + "name": "secure_tool", + "args": { + "username": "ab", # Too short (needs 3-20) + "email": "not-an-email", # Invalid format + }, + "id": "call_secure_1", + } + ], + [], + ] + ) + + agent = create_agent( + model=model, + tools=[secure_tool], + state_schema=StateWithSecrets, + ) + + result = agent.invoke( + { + "messages": [HumanMessage("Create account")], + "password": "super_secret_password_12345", + } + ) + + tool_messages = [m for m in result["messages"] if m.type == "tool"] + assert len(tool_messages) == 1 + content = tool_messages[0].content + + # The error should clearly show issues with username and/or email + # Note: validation might not catch all our expected errors since we're not + # using custom validators, but it should at least show the params + assert "username" in content.lower() or "email" in content.lower(), ( + "Error should mention at least one of the invalid parameters" + ) + + # Critical: password should NEVER appear + assert "password" not in content.lower(), ( + "Error should NOT mention 'password' (injected parameter)" + ) + assert "super_secret_password" not in content, ( + "Error should NOT contain password value from state" + ) From 73418852890cf64722817b006d962e621a4c4436 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 11:47:50 -0400 Subject: [PATCH 05/10] x --- .../langchain_v1/langchain/tools/tool_node.py | 16 ++- ...st_tool_node_validation_error_filtering.py | 113 +++++++++++------- 2 files changed, 82 insertions(+), 47 deletions(-) diff --git a/libs/langchain_v1/langchain/tools/tool_node.py b/libs/langchain_v1/langchain/tools/tool_node.py index 60c5ee47b0c70..511a8df85093e 100644 --- a/libs/langchain_v1/langchain/tools/tool_node.py +++ b/libs/langchain_v1/langchain/tools/tool_node.py @@ -471,11 +471,15 @@ def _filter_validation_errors( tool_to_store_arg: str | None, tool_to_runtime_arg: str | None, ) -> list[ErrorDetails]: - """Filter validation errors to exclude injected arguments. + """Filter validation errors to only include model-controlled arguments. - When a tool is invoked with arguments that fail validation, we need to filter - out any errors associated with injected arguments (state, store, runtime) since - these are not provided by the model and should not be reported as invocation errors. + When a tool invocation fails validation, only errors for arguments that the model + controls should be included in error messages. Injected arguments (state, store, + runtime) are provided by the system, not the model, so validation errors for them + are filtered out. + + This function also removes injected argument values from the `input` field in error + details, ensuring that only model-provided arguments appear in error messages. Args: validation_error: The Pydantic ValidationError raised during tool invocation. @@ -484,8 +488,8 @@ def _filter_validation_errors( tool_to_runtime_arg: Name of the runtime argument, if any. Returns: - List of ErrorDetails with injected argument errors filtered out and - injected values removed from input_value fields. + List of ErrorDetails containing only errors for model-controlled arguments, + with injected argument values removed from the input field. """ injected_args = set(tool_to_state_args.keys()) if tool_to_store_arg: diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py index 67d497b092821..3d04d37a857c9 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py @@ -1,8 +1,9 @@ """Unit tests for ValidationError filtering in ToolNode. -This module tests that validation errors for injected arguments (InjectedState, -InjectedStore, ToolRuntime) are properly filtered out when tools are invoked with -invalid arguments, ensuring only model-controllable argument errors are reported. +This module tests that validation errors are filtered to only include arguments +that the model controls. Injected arguments (InjectedState, InjectedStore, +ToolRuntime) are automatically provided by the system and should not appear in +validation error messages since the model has no control over them. """ from typing import Annotated @@ -42,7 +43,11 @@ def _create_config_with_runtime(store: BaseStore | None = None) -> RunnableConfi async def test_filter_injected_state_validation_errors() -> None: - """Test that validation errors for InjectedState arguments are filtered out.""" + """Test that validation errors for InjectedState arguments are filtered out. + + InjectedState parameters are not controlled by the model, so any validation + errors related to them should not appear in error messages. + """ @dec_tool def my_tool( @@ -91,7 +96,11 @@ def my_tool( async def test_filter_injected_store_validation_errors() -> None: - """Test that validation errors for InjectedStore arguments are filtered out.""" + """Test that validation errors for InjectedStore arguments are filtered out. + + InjectedStore parameters are not controlled by the model, so any validation + errors related to them should not appear in error messages. + """ @dec_tool def my_tool( @@ -145,7 +154,11 @@ def my_tool( async def test_filter_tool_runtime_validation_errors() -> None: - """Test that validation errors for ToolRuntime arguments are filtered out.""" + """Test that validation errors for ToolRuntime arguments are filtered out. + + ToolRuntime parameters are not controlled by the model, so any validation + errors related to them should not appear in error messages. + """ @dec_tool def my_tool( @@ -193,7 +206,11 @@ def my_tool( async def test_filter_multiple_injected_args() -> None: - """Test filtering when multiple injected arguments have validation errors.""" + """Test filtering when a tool has multiple injected arguments. + + When a tool uses multiple injected parameters (state, store, runtime), none of + them should appear in validation error messages since they're all system-provided. + """ @dec_tool def my_tool( @@ -246,7 +263,11 @@ def my_tool( async def test_no_filtering_when_all_errors_are_model_args() -> None: - """Test that filtering doesn't hide errors for non-injected arguments.""" + """Test that validation errors for model-controlled arguments are preserved. + + When validation fails for arguments the model controls, those errors should + be fully reported to help the model correct its tool calls. + """ @dec_tool def my_tool( @@ -299,7 +320,11 @@ def my_tool( async def test_validation_error_with_no_injected_args() -> None: - """Test that normal tools without injection still show all errors.""" + """Test that tools without injected arguments show all validation errors. + + For tools that only have model-controlled parameters, all validation errors + should be reported since everything is under the model's control. + """ @dec_tool def my_tool(value1: int, value2: str) -> str: @@ -341,7 +366,11 @@ def my_tool(value1: int, value2: str) -> str: async def test_tool_invocation_error_without_handle_errors() -> None: - """Test that ToolInvocationError is raised with filtered errors when not handling.""" + """Test that ToolInvocationError contains only model-controlled parameter errors. + + When handle_tool_errors is False, the raised ToolInvocationError should still + filter out injected arguments from the error details. + """ @dec_tool def my_tool( @@ -391,7 +420,11 @@ def my_tool( async def test_sync_tool_validation_error_filtering() -> None: - """Test that error filtering works for sync tools as well.""" + """Test that error filtering works for sync tools. + + Error filtering should work identically for both sync and async tool execution, + excluding injected arguments from validation error messages. + """ @dec_tool def my_tool( @@ -435,16 +468,16 @@ def my_tool( async def test_create_agent_error_content_with_multiple_params() -> None: - """Test that error messages contain non-injected params but not injected ones. + """Test that error messages only include model-controlled parameter errors. - This test uses create_agent to verify that when a tool with both regular + Uses create_agent to verify that when a tool with both model-controlled and injected parameters receives invalid arguments, the error message: - 1. Contains details about the non-injected parameter errors - 2. Does NOT contain any injected parameter names or values - 3. Properly formats the validation errors for clarity + 1. Contains details about model-controlled parameter errors (query, limit) + 2. Does NOT contain injected parameter names (state, store, runtime) + 3. Does NOT contain values from injected parameters + 4. Properly formats the validation errors for model correction """ - # Custom state with sensitive information class TestState(AgentState): user_id: str api_key: str @@ -519,11 +552,8 @@ def complex_tool( content = tool_message.content - # Verify error mentions the non-injected parameter issues - # Should mention 'query' error + # Verify error mentions model-controlled parameter issues assert "query" in content.lower(), "Error should mention 'query' parameter" - - # Should mention 'limit' error (missing required field) assert "limit" in content.lower(), "Error should mention 'limit' parameter" # Should indicate validation errors occurred @@ -531,34 +561,34 @@ def complex_tool( "Error should indicate validation occurred" ) - # CRITICAL: Verify NO injected parameter names appear in error + # Verify NO injected parameter names appear in error + # These are not controlled by the model and should be excluded assert "state" not in content.lower(), "Error should NOT mention 'state' (injected parameter)" assert "store" not in content.lower(), "Error should NOT mention 'store' (injected parameter)" assert "runtime" not in content.lower(), ( "Error should NOT mention 'runtime' (injected parameter)" ) - # CRITICAL: Verify NO sensitive values from state appear in error - assert "user_12345" not in content, "Error should NOT contain user_id value from state" - assert "sk-secret-key" not in content, "Error should NOT contain api_key value from state" - assert "secret_session_token" not in content, "Error should NOT contain session data from state" + # Verify NO values from injected parameters appear in error + # The model doesn't control these, so they shouldn't be in the error message + assert "user_12345" not in content, "Error should NOT contain user_id value" + assert "sk-secret-key" not in content, "Error should NOT contain api_key value" + assert "secret_session_token" not in content, "Error should NOT contain session_data value" - # Verify the original tool call args are mentioned (not the injected ones) - # The error template includes: "with kwargs {tool_kwargs}" - # This should show the original args from the model's tool call + # Verify the original model tool call args are present + # The error should show what the model actually provided assert "12345" in content, "Error should show the invalid query value (12345)" - # Additional verification: check error structure - # Should be formatted in a readable way + # Check error is well-formatted assert "complex_tool" in content, "Error should mention the tool name" async def test_create_agent_error_only_model_controllable_params() -> None: - """Test that errors only show model-controllable parameter issues. + """Test that errors only include model-controllable parameter issues. - This is a focused test ensuring that when ONLY non-injected parameters - have validation errors, those errors are clearly shown without any - confusion from injected parameters. + Focused test ensuring that validation errors for model-controlled parameters + are clearly reported, while injected parameters remain completely absent from + error messages. """ class StateWithSecrets(AgentState): @@ -613,17 +643,18 @@ def secure_tool( assert len(tool_messages) == 1 content = tool_messages[0].content - # The error should clearly show issues with username and/or email - # Note: validation might not catch all our expected errors since we're not - # using custom validators, but it should at least show the params + # The error should mention model-controlled parameters + # Note: Pydantic's default validation may or may not catch format issues, + # but the parameters themselves should be present in error messages assert "username" in content.lower() or "email" in content.lower(), ( - "Error should mention at least one of the invalid parameters" + "Error should mention at least one model-controlled parameter" ) - # Critical: password should NEVER appear + # Password is injected and should not appear + # The model doesn't control it, so it shouldn't be in the error assert "password" not in content.lower(), ( "Error should NOT mention 'password' (injected parameter)" ) assert "super_secret_password" not in content, ( - "Error should NOT contain password value from state" + "Error should NOT contain password value (from injected state)" ) From d2d8588cccf81e38b5432a5dcd72dd6551f1bf57 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 12:24:14 -0400 Subject: [PATCH 06/10] x --- .../tests/unit_tests/agents/test_tool_node.py | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py index 7168916a786fb..78ec32eb1455f 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py @@ -368,6 +368,97 @@ def tool_with_injected_state( assert "sensitive_secret_123" not in tool_message.content +async def test_tool_invocation_error_excludes_injected_state_async() -> None: + """Test that async tool invocation errors exclude injected state information. + + This test verifies that the async execution path (_execute_tool_async and _arun_one) + properly filters validation errors to exclude injected arguments, just like the + sync path does. + """ + + # Define a custom state schema + class TestState(AgentState): + internal_data: str + + @dec_tool + async def async_tool_with_injected_state( + query: str, + max_results: int, + state: Annotated[TestState, InjectedState], + ) -> str: + """Async tool that uses injected state.""" + return f"query: {query}, max_results: {max_results}" + + # Create a fake model that makes an incorrect tool call + # - query has wrong type (int instead of str) + # - max_results is missing + model = FakeToolCallingModel( + tool_calls=[ + [ + { + "name": "async_tool_with_injected_state", + "args": {"query": 999}, # Wrong type, missing max_results + "id": "call_async_1", + } + ], + [], # End the loop + ] + ) + + # Create an agent with the async tool + agent = create_agent( + model=model, + tools=[async_tool_with_injected_state], + state_schema=TestState, + ) + + # Invoke with state data + result = await agent.ainvoke( + { + "messages": [HumanMessage("Test async")], + "internal_data": "secret_internal_value_xyz", + } + ) + + # Find the tool error message + tool_messages = [m for m in result["messages"] if m.type == "tool"] + assert len(tool_messages) == 1 + tool_message = tool_messages[0] + assert tool_message.status == "error" + + # Verify error mentions model-controlled parameters + content = tool_message.content + assert "query" in content.lower(), "Error should mention 'query'" + assert "max_results" in content.lower(), "Error should mention 'max_results'" + + # Verify injected state does not appear in the validation errors + # The tool name may contain "state", but the error list should not mention it + assert "internal_data" not in content, ( + "Error should NOT mention 'internal_data' field from state" + ) + assert "secret_internal_value" not in content, "Error should NOT contain state values" + + # Verify only the model-controlled parameters are in the error list + # Should see "query" and "max_results" errors, but not "state" + lines = content.split("\n") + error_lines = [line.strip() for line in lines if line.strip()] + # Find lines that look like field names (single words at start of line) + field_errors = [ + line + for line in error_lines + if line + and not line.startswith("input") + and not line.startswith("field") + and not line.startswith("error") + and not line.startswith("please") + and len(line.split()) <= 2 + ] + # Check that state is not in the field error list + assert not any("state" == field.lower() for field in field_errors), ( + "The field 'state' should not appear in validation errors" + ) + + async def test_tool_node_error_handling() -> None: def handle_all(e: ValueError | ToolException | ToolInvocationError): return TOOL_CALL_ERROR_TEMPLATE.format(error=repr(e)) From 5067be1760cacf47355e5f90807a5df7297f86ca Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 12:41:09 -0400 Subject: [PATCH 07/10] x --- .../langchain_v1/langchain/tools/tool_node.py | 17 +-- .../tests/unit_tests/agents/test_tool_node.py | 43 ++++--- ...st_tool_node_validation_error_filtering.py | 117 ++++++++++-------- 3 files changed, 97 insertions(+), 80 deletions(-) diff --git a/libs/langchain_v1/langchain/tools/tool_node.py b/libs/langchain_v1/langchain/tools/tool_node.py index 511a8df85093e..48e4fe597a852 100644 --- a/libs/langchain_v1/langchain/tools/tool_node.py +++ b/libs/langchain_v1/langchain/tools/tool_node.py @@ -471,15 +471,16 @@ def _filter_validation_errors( tool_to_store_arg: str | None, tool_to_runtime_arg: str | None, ) -> list[ErrorDetails]: - """Filter validation errors to only include model-controlled arguments. + """Filter validation errors to only include LLM-controlled arguments. - When a tool invocation fails validation, only errors for arguments that the model - controls should be included in error messages. Injected arguments (state, store, - runtime) are provided by the system, not the model, so validation errors for them - are filtered out. + When a tool invocation fails validation, only errors for arguments that the LLM + controls should be included in error messages. This ensures the LLM receives + focused, actionable feedback about parameters it can actually fix. System-injected + arguments (state, store, runtime) are filtered out since the LLM has no control + over them. This function also removes injected argument values from the `input` field in error - details, ensuring that only model-provided arguments appear in error messages. + details, ensuring that only LLM-provided arguments appear in error messages. Args: validation_error: The Pydantic ValidationError raised during tool invocation. @@ -488,8 +489,8 @@ def _filter_validation_errors( tool_to_runtime_arg: Name of the runtime argument, if any. Returns: - List of ErrorDetails containing only errors for model-controlled arguments, - with injected argument values removed from the input field. + List of ErrorDetails containing only errors for LLM-controlled arguments, + with system-injected argument values removed from the input field. """ injected_args = set(tool_to_state_args.keys()) if tool_to_store_arg: diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py index 78ec32eb1455f..9bd3747d60695 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py @@ -305,18 +305,20 @@ def test_tool_node_error_handling_default_exception() -> None: def test_tool_invocation_error_excludes_injected_state() -> None: - """Test that tool invocation errors do not include injected state information. + """Test that tool invocation errors only include LLM-controllable arguments. When a tool has InjectedState parameters and the LLM makes an incorrect invocation (e.g., missing required arguments), the error message should only - contain the original arguments from the tool call, not the injected state values. + contain the arguments from the tool call that the LLM controls. This ensures + the LLM receives relevant context to correct its mistakes, without being + distracted by system-injected parameters it has no control over. This test uses create_agent to ensure the behavior works in a full agent context. """ - # Define a custom state schema with secret data + # Define a custom state schema with injected data class TestState(AgentState): - secret_data: str + secret_data: str # Example of state data not controlled by LLM @dec_tool def tool_with_injected_state( @@ -348,7 +350,7 @@ def tool_with_injected_state( state_schema=TestState, ) - # Invoke the agent with secret data in the state + # Invoke the agent with injected state data result = agent.invoke( { "messages": [HumanMessage("Test message")], @@ -362,18 +364,19 @@ def tool_with_injected_state( tool_message = tool_messages[0] assert tool_message.status == "error" - # The error message should contain the original args in the kwargs section + # The error message should contain only the LLM-provided args (wrong_arg) + # and NOT the system-injected state (secret_data) assert "{'wrong_arg': 'value'}" in tool_message.content assert "secret_data" not in tool_message.content assert "sensitive_secret_123" not in tool_message.content async def test_tool_invocation_error_excludes_injected_state_async() -> None: - """Test that async tool invocation errors exclude injected state information. + """Test that async tool invocation errors only include LLM-controllable arguments. This test verifies that the async execution path (_execute_tool_async and _arun_one) - properly filters validation errors to exclude injected arguments, just like the - sync path does. + properly filters validation errors to exclude system-injected arguments, ensuring + the LLM receives only relevant context for correction. """ # Define a custom state schema @@ -426,19 +429,21 @@ async def async_tool_with_injected_state( tool_message = tool_messages[0] assert tool_message.status == "error" - # Verify error mentions model-controlled parameters + # Verify error mentions LLM-controlled parameters only content = tool_message.content - assert "query" in content.lower(), "Error should mention 'query'" - assert "max_results" in content.lower(), "Error should mention 'max_results'" + assert "query" in content.lower(), "Error should mention 'query' (LLM-controlled)" + assert "max_results" in content.lower(), "Error should mention 'max_results' (LLM-controlled)" - # Verify injected state does not appear in the validation errors - # The tool name may contain "state", but the error list should not mention it + # Verify system-injected state does not appear in the validation errors + # This keeps the error focused on what the LLM can actually fix assert "internal_data" not in content, ( - "Error should NOT mention 'internal_data' field from state" + "Error should NOT mention 'internal_data' (system-injected field)" + ) + assert "secret_internal_value" not in content, ( + "Error should NOT contain system-injected state values" ) - assert "secret_internal_value" not in content, "Error should NOT contain state values" - # Verify only the model-controlled parameters are in the error list + # Verify only LLM-controlled parameters are in the error list # Should see "query" and "max_results" errors, but not "state" lines = content.split("\n") error_lines = [line.strip() for line in lines if line.strip()] @@ -453,9 +458,9 @@ async def async_tool_with_injected_state( and not line.startswith("please") and len(line.split()) <= 2 ] - # Check that state is not in the field error list + # Verify system-injected 'state' is not in the field error list assert not any("state" == field.lower() for field in field_errors), ( - "The field 'state' should not appear in validation errors" + "The field 'state' (system-injected) should not appear in validation errors" ) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py index 3d04d37a857c9..3576c89cb3f65 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py @@ -1,9 +1,11 @@ """Unit tests for ValidationError filtering in ToolNode. This module tests that validation errors are filtered to only include arguments -that the model controls. Injected arguments (InjectedState, InjectedStore, +that the LLM controls. Injected arguments (InjectedState, InjectedStore, ToolRuntime) are automatically provided by the system and should not appear in -validation error messages since the model has no control over them. +validation error messages. This ensures the LLM receives focused, actionable +feedback about the parameters it can actually control, improving error correction +and reducing confusion from irrelevant system implementation details. """ from typing import Annotated @@ -45,8 +47,9 @@ def _create_config_with_runtime(store: BaseStore | None = None) -> RunnableConfi async def test_filter_injected_state_validation_errors() -> None: """Test that validation errors for InjectedState arguments are filtered out. - InjectedState parameters are not controlled by the model, so any validation - errors related to them should not appear in error messages. + InjectedState parameters are not controlled by the LLM, so any validation + errors related to them should not appear in error messages. This ensures + the LLM receives only actionable feedback about its own tool call arguments. """ @dec_tool @@ -98,8 +101,9 @@ def my_tool( async def test_filter_injected_store_validation_errors() -> None: """Test that validation errors for InjectedStore arguments are filtered out. - InjectedStore parameters are not controlled by the model, so any validation - errors related to them should not appear in error messages. + InjectedStore parameters are not controlled by the LLM, so any validation + errors related to them should not appear in error messages. This keeps + error feedback focused on LLM-controllable parameters. """ @dec_tool @@ -156,8 +160,9 @@ def my_tool( async def test_filter_tool_runtime_validation_errors() -> None: """Test that validation errors for ToolRuntime arguments are filtered out. - ToolRuntime parameters are not controlled by the model, so any validation - errors related to them should not appear in error messages. + ToolRuntime parameters are not controlled by the LLM, so any validation + errors related to them should not appear in error messages. This ensures + the LLM only sees errors for parameters it can fix. """ @dec_tool @@ -209,7 +214,8 @@ async def test_filter_multiple_injected_args() -> None: """Test filtering when a tool has multiple injected arguments. When a tool uses multiple injected parameters (state, store, runtime), none of - them should appear in validation error messages since they're all system-provided. + them should appear in validation error messages since they're all system-provided + and not controlled by the LLM. Only LLM-controllable parameter errors should appear. """ @dec_tool @@ -263,10 +269,11 @@ def my_tool( async def test_no_filtering_when_all_errors_are_model_args() -> None: - """Test that validation errors for model-controlled arguments are preserved. + """Test that validation errors for LLM-controlled arguments are preserved. - When validation fails for arguments the model controls, those errors should - be fully reported to help the model correct its tool calls. + When validation fails for arguments the LLM controls, those errors should + be fully reported to help the LLM correct its tool calls. This ensures + the LLM receives complete feedback about all issues it can fix. """ @dec_tool @@ -322,8 +329,9 @@ def my_tool( async def test_validation_error_with_no_injected_args() -> None: """Test that tools without injected arguments show all validation errors. - For tools that only have model-controlled parameters, all validation errors - should be reported since everything is under the model's control. + For tools that only have LLM-controlled parameters, all validation errors + should be reported since everything is under the LLM's control and can be + corrected by the LLM in subsequent tool calls. """ @dec_tool @@ -366,10 +374,11 @@ def my_tool(value1: int, value2: str) -> str: async def test_tool_invocation_error_without_handle_errors() -> None: - """Test that ToolInvocationError contains only model-controlled parameter errors. + """Test that ToolInvocationError contains only LLM-controlled parameter errors. When handle_tool_errors is False, the raised ToolInvocationError should still - filter out injected arguments from the error details. + filter out system-injected arguments from the error details, ensuring that + error messages focus on what the LLM can control. """ @dec_tool @@ -468,14 +477,16 @@ def my_tool( async def test_create_agent_error_content_with_multiple_params() -> None: - """Test that error messages only include model-controlled parameter errors. - - Uses create_agent to verify that when a tool with both model-controlled - and injected parameters receives invalid arguments, the error message: - 1. Contains details about model-controlled parameter errors (query, limit) - 2. Does NOT contain injected parameter names (state, store, runtime) - 3. Does NOT contain values from injected parameters - 4. Properly formats the validation errors for model correction + """Test that error messages only include LLM-controlled parameter errors. + + Uses create_agent to verify that when a tool with both LLM-controlled + and system-injected parameters receives invalid arguments, the error message: + 1. Contains details about LLM-controlled parameter errors (query, limit) + 2. Does NOT contain system-injected parameter names (state, store, runtime) + 3. Does NOT contain values from system-injected parameters + 4. Properly formats the validation errors for LLM correction + + This ensures the LLM receives focused, actionable feedback. """ class TestState(AgentState): @@ -552,47 +563,47 @@ def complex_tool( content = tool_message.content - # Verify error mentions model-controlled parameter issues - assert "query" in content.lower(), "Error should mention 'query' parameter" - assert "limit" in content.lower(), "Error should mention 'limit' parameter" + # Verify error mentions LLM-controlled parameter issues + assert "query" in content.lower(), "Error should mention 'query' (LLM-controlled)" + assert "limit" in content.lower(), "Error should mention 'limit' (LLM-controlled)" # Should indicate validation errors occurred assert "validation error" in content.lower() or "error" in content.lower(), ( "Error should indicate validation occurred" ) - # Verify NO injected parameter names appear in error - # These are not controlled by the model and should be excluded - assert "state" not in content.lower(), "Error should NOT mention 'state' (injected parameter)" - assert "store" not in content.lower(), "Error should NOT mention 'store' (injected parameter)" + # Verify NO system-injected parameter names appear in error + # These are not controlled by the LLM and should be excluded + assert "state" not in content.lower(), "Error should NOT mention 'state' (system-injected)" + assert "store" not in content.lower(), "Error should NOT mention 'store' (system-injected)" assert "runtime" not in content.lower(), ( - "Error should NOT mention 'runtime' (injected parameter)" + "Error should NOT mention 'runtime' (system-injected)" ) - # Verify NO values from injected parameters appear in error - # The model doesn't control these, so they shouldn't be in the error message - assert "user_12345" not in content, "Error should NOT contain user_id value" - assert "sk-secret-key" not in content, "Error should NOT contain api_key value" - assert "secret_session_token" not in content, "Error should NOT contain session_data value" + # Verify NO values from system-injected parameters appear in error + # The LLM doesn't control these, so they shouldn't distract from the actual issues + assert "user_12345" not in content, "Error should NOT contain user_id value (from state)" + assert "sk-secret-key" not in content, "Error should NOT contain api_key value (from state)" + assert "secret_session_token" not in content, "Error should NOT contain session_data value (from state)" - # Verify the original model tool call args are present - # The error should show what the model actually provided - assert "12345" in content, "Error should show the invalid query value (12345)" + # Verify the LLM's original tool call args are present + # The error should show what the LLM actually provided to help it correct the mistake + assert "12345" in content, "Error should show the invalid query value provided by LLM (12345)" # Check error is well-formatted assert "complex_tool" in content, "Error should mention the tool name" async def test_create_agent_error_only_model_controllable_params() -> None: - """Test that errors only include model-controllable parameter issues. + """Test that errors only include LLM-controllable parameter issues. - Focused test ensuring that validation errors for model-controlled parameters - are clearly reported, while injected parameters remain completely absent from - error messages. + Focused test ensuring that validation errors for LLM-controlled parameters + are clearly reported, while system-injected parameters remain completely + absent from error messages. This provides focused feedback to the LLM. """ class StateWithSecrets(AgentState): - password: str + password: str # Example of data not controlled by LLM @dec_tool def secure_tool( @@ -605,11 +616,11 @@ def secure_tool( Args: username: The username (3-20 chars). email: The email address. - state: State with password (injected). + state: State with password (system-injected). """ return f"Validated {username} with email {email}" - # Model provides invalid username (too short) and invalid email + # LLM provides invalid username (too short) and invalid email model = FakeToolCallingModel( tool_calls=[ [ @@ -643,18 +654,18 @@ def secure_tool( assert len(tool_messages) == 1 content = tool_messages[0].content - # The error should mention model-controlled parameters + # The error should mention LLM-controlled parameters # Note: Pydantic's default validation may or may not catch format issues, # but the parameters themselves should be present in error messages assert "username" in content.lower() or "email" in content.lower(), ( - "Error should mention at least one model-controlled parameter" + "Error should mention at least one LLM-controlled parameter" ) - # Password is injected and should not appear - # The model doesn't control it, so it shouldn't be in the error + # Password is system-injected and should not appear + # The LLM doesn't control it, so it shouldn't distract from the actual errors assert "password" not in content.lower(), ( - "Error should NOT mention 'password' (injected parameter)" + "Error should NOT mention 'password' (system-injected parameter)" ) assert "super_secret_password" not in content, ( - "Error should NOT contain password value (from injected state)" + "Error should NOT contain password value (from system-injected state)" ) From 6829386f5004988989b77b39b83647bbbbb1d946 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 12:50:06 -0400 Subject: [PATCH 08/10] x --- .../agents/test_tool_node_validation_error_filtering.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py index 3576c89cb3f65..fb7f23e559a6a 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node_validation_error_filtering.py @@ -576,15 +576,15 @@ def complex_tool( # These are not controlled by the LLM and should be excluded assert "state" not in content.lower(), "Error should NOT mention 'state' (system-injected)" assert "store" not in content.lower(), "Error should NOT mention 'store' (system-injected)" - assert "runtime" not in content.lower(), ( - "Error should NOT mention 'runtime' (system-injected)" - ) + assert "runtime" not in content.lower(), "Error should NOT mention 'runtime' (system-injected)" # Verify NO values from system-injected parameters appear in error # The LLM doesn't control these, so they shouldn't distract from the actual issues assert "user_12345" not in content, "Error should NOT contain user_id value (from state)" assert "sk-secret-key" not in content, "Error should NOT contain api_key value (from state)" - assert "secret_session_token" not in content, "Error should NOT contain session_data value (from state)" + assert "secret_session_token" not in content, ( + "Error should NOT contain session_data value (from state)" + ) # Verify the LLM's original tool call args are present # The error should show what the LLM actually provided to help it correct the mistake From 0380e50d311bdb385c50d5bb7f2ed0285d6a80b2 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 12:58:52 -0400 Subject: [PATCH 09/10] x --- .../langchain_v1/langchain/tools/tool_node.py | 100 +----------------- 1 file changed, 4 insertions(+), 96 deletions(-) diff --git a/libs/langchain_v1/langchain/tools/tool_node.py b/libs/langchain_v1/langchain/tools/tool_node.py index 48e4fe597a852..89a1b92892ccc 100644 --- a/libs/langchain_v1/langchain/tools/tool_node.py +++ b/libs/langchain_v1/langchain/tools/tool_node.py @@ -89,7 +89,6 @@ def my_tool(x: int) -> str: from collections.abc import Sequence from langgraph.runtime import Runtime - from pydantic_core import ErrorDetails # right now we use a dict as the default, can change this to AgentState, but depends # on if this lives in LangChain or LangGraph... ideally would have some typed @@ -304,11 +303,7 @@ class ToolInvocationError(ToolException): """ def __init__( - self, - tool_name: str, - source: ValidationError, - tool_kwargs: dict[str, Any], - filtered_errors: list[ErrorDetails] | None = None, + self, tool_name: str, source: ValidationError, tool_kwargs: dict[str, Any] ) -> None: """Initialize the ToolInvocationError. @@ -316,31 +311,13 @@ def __init__( tool_name: The name of the tool that failed. source: The exception that occurred. tool_kwargs: The keyword arguments that were passed to the tool. - filtered_errors: Optional list of filtered validation errors excluding - injected arguments. """ - # Format error display based on filtered errors if provided - if filtered_errors is not None: - # Manually format the filtered errors - error_count = len(filtered_errors) - plural = "s" if error_count != 1 else "" - header = f"{error_count} validation error{plural} for {tool_name}" - error_str_parts = [header] - for error in filtered_errors: - loc_str = " -> ".join(str(loc) for loc in error.get("loc", ())) - error_str_parts.append(f"{loc_str}") - error_str_parts.append(f" {error.get('msg', 'Unknown error')}") - error_display_str = "\n".join(error_str_parts) - else: - error_display_str = str(source) - self.message = TOOL_INVOCATION_ERROR_TEMPLATE.format( - tool_name=tool_name, tool_kwargs=tool_kwargs, error=error_display_str + tool_name=tool_name, tool_kwargs=tool_kwargs, error=source ) self.tool_name = tool_name self.tool_kwargs = tool_kwargs self.source = source - self.filtered_errors = filtered_errors super().__init__(self.message) @@ -465,59 +442,6 @@ def _infer_handled_types(handler: Callable[..., str]) -> tuple[type[Exception], return (Exception,) -def _filter_validation_errors( - validation_error: ValidationError, - tool_to_state_args: dict[str, str | None], - tool_to_store_arg: str | None, - tool_to_runtime_arg: str | None, -) -> list[ErrorDetails]: - """Filter validation errors to only include LLM-controlled arguments. - - When a tool invocation fails validation, only errors for arguments that the LLM - controls should be included in error messages. This ensures the LLM receives - focused, actionable feedback about parameters it can actually fix. System-injected - arguments (state, store, runtime) are filtered out since the LLM has no control - over them. - - This function also removes injected argument values from the `input` field in error - details, ensuring that only LLM-provided arguments appear in error messages. - - Args: - validation_error: The Pydantic ValidationError raised during tool invocation. - tool_to_state_args: Mapping of state argument names to state field names. - tool_to_store_arg: Name of the store argument, if any. - tool_to_runtime_arg: Name of the runtime argument, if any. - - Returns: - List of ErrorDetails containing only errors for LLM-controlled arguments, - with system-injected argument values removed from the input field. - """ - injected_args = set(tool_to_state_args.keys()) - if tool_to_store_arg: - injected_args.add(tool_to_store_arg) - if tool_to_runtime_arg: - injected_args.add(tool_to_runtime_arg) - - filtered_errors: list[ErrorDetails] = [] - for error in validation_error.errors(): - # Check if error location contains any injected argument - # error['loc'] is a tuple like ('field_name',) or ('field_name', 'nested_field') - if error["loc"] and error["loc"][0] not in injected_args: - # Create a copy of the error dict to avoid mutating the original - error_copy: dict[str, Any] = {**error} - - # Remove injected arguments from input_value if it's a dict - if isinstance(error_copy.get("input"), dict): - input_dict = error_copy["input"] - input_copy = {k: v for k, v in input_dict.items() if k not in injected_args} - error_copy["input"] = input_copy - - # Cast is safe because ErrorDetails is a TypedDict compatible with this structure - filtered_errors.append(error_copy) # type: ignore[arg-type] - - return filtered_errors - - class _ToolNode(RunnableCallable): """A node for executing tools in LangGraph workflows. @@ -699,7 +623,6 @@ def _func( ) tool_runtimes.append(tool_runtime) - # Pass original tool calls without injection input_types = [input_type] * len(tool_calls) with get_executor_for_config(config) as executor: outputs = list(executor.map(self._run_one, tool_calls, input_types, tool_runtimes)) @@ -729,7 +652,6 @@ async def _afunc( ) tool_runtimes.append(tool_runtime) - # Pass original tool calls without injection coros = [] for call, tool_runtime in zip(tool_calls, tool_runtimes, strict=False): coros.append(self._arun_one(call, input_type, tool_runtime)) # type: ignore[arg-type] @@ -817,15 +739,8 @@ def _execute_tool_sync( try: response = tool.invoke(call_args, config) except ValidationError as exc: - # Filter out errors for injected arguments - filtered_errors = _filter_validation_errors( - exc, - self._tool_to_state_args.get(call["name"], {}), - self._tool_to_store_arg.get(call["name"]), - self._tool_to_runtime_arg.get(call["name"]), - ) # Use original call["args"] without injected values for error reporting - raise ToolInvocationError(call["name"], exc, call["args"], filtered_errors) from exc + raise ToolInvocationError(call["name"], exc, call["args"]) from exc # GraphInterrupt is a special exception that will always be raised. # It can be triggered in the following scenarios, @@ -972,15 +887,8 @@ async def _execute_tool_async( try: response = await tool.ainvoke(call_args, config) except ValidationError as exc: - # Filter out errors for injected arguments - filtered_errors = _filter_validation_errors( - exc, - self._tool_to_state_args.get(call["name"], {}), - self._tool_to_store_arg.get(call["name"]), - self._tool_to_runtime_arg.get(call["name"]), - ) # Use original call["args"] without injected values for error reporting - raise ToolInvocationError(call["name"], exc, call["args"], filtered_errors) from exc + raise ToolInvocationError(call["name"], exc, call["args"]) from exc # GraphInterrupt is a special exception that will always be raised. # It can be triggered in the following scenarios, From ef1f031801255b8cfc59ee66c650f1f9c26d9d54 Mon Sep 17 00:00:00 2001 From: Eugene Yurtsev Date: Tue, 21 Oct 2025 12:59:03 -0400 Subject: [PATCH 10/10] ux --- .../tests/unit_tests/agents/test_tool_node.py | 162 ------------------ 1 file changed, 162 deletions(-) diff --git a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py index 9bd3747d60695..0f04f98832b28 100644 --- a/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py +++ b/libs/langchain_v1/tests/unit_tests/agents/test_tool_node.py @@ -10,8 +10,6 @@ TypeVar, ) from unittest.mock import Mock -from langchain.agents import create_agent -from langchain.agents.middleware.types import AgentState import pytest from langchain_core.messages import ( @@ -304,166 +302,6 @@ def test_tool_node_error_handling_default_exception() -> None: ) -def test_tool_invocation_error_excludes_injected_state() -> None: - """Test that tool invocation errors only include LLM-controllable arguments. - - When a tool has InjectedState parameters and the LLM makes an incorrect - invocation (e.g., missing required arguments), the error message should only - contain the arguments from the tool call that the LLM controls. This ensures - the LLM receives relevant context to correct its mistakes, without being - distracted by system-injected parameters it has no control over. - - This test uses create_agent to ensure the behavior works in a full agent context. - """ - - # Define a custom state schema with injected data - class TestState(AgentState): - secret_data: str # Example of state data not controlled by LLM - - @dec_tool - def tool_with_injected_state( - some_val: int, - state: Annotated[TestState, InjectedState], - ) -> str: - """Tool that uses injected state.""" - return f"some_val: {some_val}" - - # Create a fake model that makes an incorrect tool call (missing 'some_val') - # Then returns no tool calls on the second iteration to end the loop - model = FakeToolCallingModel( - tool_calls=[ - [ - { - "name": "tool_with_injected_state", - "args": {"wrong_arg": "value"}, # Missing required 'some_val' - "id": "call_1", - } - ], - [], # No tool calls on second iteration to end the loop - ] - ) - - # Create an agent with the tool and custom state schema - agent = create_agent( - model=model, - tools=[tool_with_injected_state], - state_schema=TestState, - ) - - # Invoke the agent with injected state data - result = agent.invoke( - { - "messages": [HumanMessage("Test message")], - "secret_data": "sensitive_secret_123", - } - ) - - # Find the tool error message - tool_messages = [m for m in result["messages"] if m.type == "tool"] - assert len(tool_messages) == 1 - tool_message = tool_messages[0] - assert tool_message.status == "error" - - # The error message should contain only the LLM-provided args (wrong_arg) - # and NOT the system-injected state (secret_data) - assert "{'wrong_arg': 'value'}" in tool_message.content - assert "secret_data" not in tool_message.content - assert "sensitive_secret_123" not in tool_message.content - - -async def test_tool_invocation_error_excludes_injected_state_async() -> None: - """Test that async tool invocation errors only include LLM-controllable arguments. - - This test verifies that the async execution path (_execute_tool_async and _arun_one) - properly filters validation errors to exclude system-injected arguments, ensuring - the LLM receives only relevant context for correction. - """ - - # Define a custom state schema - class TestState(AgentState): - internal_data: str - - @dec_tool - async def async_tool_with_injected_state( - query: str, - max_results: int, - state: Annotated[TestState, InjectedState], - ) -> str: - """Async tool that uses injected state.""" - return f"query: {query}, max_results: {max_results}" - - # Create a fake model that makes an incorrect tool call - # - query has wrong type (int instead of str) - # - max_results is missing - model = FakeToolCallingModel( - tool_calls=[ - [ - { - "name": "async_tool_with_injected_state", - "args": {"query": 999}, # Wrong type, missing max_results - "id": "call_async_1", - } - ], - [], # End the loop - ] - ) - - # Create an agent with the async tool - agent = create_agent( - model=model, - tools=[async_tool_with_injected_state], - state_schema=TestState, - ) - - # Invoke with state data - result = await agent.ainvoke( - { - "messages": [HumanMessage("Test async")], - "internal_data": "secret_internal_value_xyz", - } - ) - - # Find the tool error message - tool_messages = [m for m in result["messages"] if m.type == "tool"] - assert len(tool_messages) == 1 - tool_message = tool_messages[0] - assert tool_message.status == "error" - - # Verify error mentions LLM-controlled parameters only - content = tool_message.content - assert "query" in content.lower(), "Error should mention 'query' (LLM-controlled)" - assert "max_results" in content.lower(), "Error should mention 'max_results' (LLM-controlled)" - - # Verify system-injected state does not appear in the validation errors - # This keeps the error focused on what the LLM can actually fix - assert "internal_data" not in content, ( - "Error should NOT mention 'internal_data' (system-injected field)" - ) - assert "secret_internal_value" not in content, ( - "Error should NOT contain system-injected state values" - ) - - # Verify only LLM-controlled parameters are in the error list - # Should see "query" and "max_results" errors, but not "state" - lines = content.split("\n") - error_lines = [line.strip() for line in lines if line.strip()] - # Find lines that look like field names (single words at start of line) - field_errors = [ - line - for line in error_lines - if line - and not line.startswith("input") - and not line.startswith("field") - and not line.startswith("error") - and not line.startswith("please") - and len(line.split()) <= 2 - ] - # Verify system-injected 'state' is not in the field error list - assert not any("state" == field.lower() for field in field_errors), ( - "The field 'state' (system-injected) should not appear in validation errors" - ) - - async def test_tool_node_error_handling() -> None: def handle_all(e: ValueError | ToolException | ToolInvocationError): return TOOL_CALL_ERROR_TEMPLATE.format(error=repr(e))