Closed
Description
What happened?
A batch request to an Ethereum RPC endpoint is responded with an array containing the responses to each request in the batch:
curl http://localhost:8545 -H "Content-Type: application/json" --data '[{"jsonrpc":"2.0", "id": 1, "method": "eth_blockNumber", "params": []}]'
[{"jsonrpc":"2.0","id":1,"result":"0x1410bb2"}]So Web3.py assumes the response is an array while parsing it:
web3.py/web3/providers/rpc/rpc.py
Lines 187 to 188 in 3c412f8
However, when an empty batch is executed, some RPC endpoint (mine is Erigon) returns an object specifying the error instead of an array. This breaks the parsing and result in error: AttributeError: 'str' object has no attribute 'get'
curl http://localhost:8545 -H "Content-Type: application/json" --data '[]'
{"jsonrpc":"2.0","id":null,"error":{"code":-32600,"message":"empty batch"}}The same happens when the batch size exceeds the limit:
{"jsonrpc":"2.0","id":null,"error":{"code":-32000,"message":"batch limit 100 exceeded (can increase by --rpc.batch.limit). Requested batch of size: 526"}}Code that produced the error
from web3 import Web3, HTTPProvider
w3 = Web3(HTTPProvider("http://localhost:8545"))
with w3.batch_requests() as batch:
batch.execute()Full error output
AttributeError Traceback (most recent call last)
Cell In[222], line 5
3 with w3.batch_requests() as batch:
----> 4 batch.execute()
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/_utils/batching.py:147, in RequestBatcher.execute(self)
145 def execute(self) -> List["RPCResponse"]:
146 self._validate_is_batching()
--> 147 responses = self.web3.manager._make_batch_request(self._requests_info)
148 self._end_batching()
149 return responses
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/manager.py:430, in RequestManager._make_batch_request(self, requests_info)
426 provider = cast(JSONBaseProvider, self.provider)
427 request_func = provider.batch_request_func(
428 cast("Web3", self.w3), cast("MiddlewareOnion", self.middleware_onion)
429 )
--> 430 responses = request_func(
431 [
432 (method, params)
433 for (method, params), _response_formatters in requests_info
434 ]
435 )
436 formatted_responses = [
437 self._format_batched_response(info, resp)
438 for info, resp in zip(requests_info, responses)
439 ]
440 return list(formatted_responses)
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/middleware/base.py:70, in Web3Middleware.wrap_make_batch_request.<locals>.middleware(requests_info)
63 def middleware(
64 requests_info: List[Tuple["RPCEndpoint", Any]]
65 ) -> List["RPCResponse"]:
66 req_processed = [
67 self.request_processor(method, params)
68 for (method, params) in requests_info
69 ]
---> 70 responses = make_batch_request(req_processed)
71 methods, _params = zip(*req_processed)
72 formatted_responses = [
73 self.response_processor(m, r) for m, r in zip(methods, responses)
74 ]
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/middleware/base.py:70, in Web3Middleware.wrap_make_batch_request.<locals>.middleware(requests_info)
63 def middleware(
64 requests_info: List[Tuple["RPCEndpoint", Any]]
65 ) -> List["RPCResponse"]:
66 req_processed = [
67 self.request_processor(method, params)
68 for (method, params) in requests_info
69 ]
---> 70 responses = make_batch_request(req_processed)
71 methods, _params = zip(*req_processed)
72 formatted_responses = [
73 self.response_processor(m, r) for m, r in zip(methods, responses)
74 ]
[... skipping similar frames: Web3Middleware.wrap_make_batch_request.<locals>.middleware at line 70 (2 times)]
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/middleware/base.py:70, in Web3Middleware.wrap_make_batch_request.<locals>.middleware(requests_info)
63 def middleware(
64 requests_info: List[Tuple["RPCEndpoint", Any]]
65 ) -> List["RPCResponse"]:
66 req_processed = [
67 self.request_processor(method, params)
68 for (method, params) in requests_info
69 ]
---> 70 responses = make_batch_request(req_processed)
71 methods, _params = zip(*req_processed)
72 formatted_responses = [
73 self.response_processor(m, r) for m, r in zip(methods, responses)
74 ]
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/providers/rpc/rpc.py:188, in HTTPProvider.make_batch_request(self, batch_requests)
186 self.logger.debug("Received batch response HTTP.")
187 responses_list = cast(List[RPCResponse], self.decode_rpc_response(raw_response))
--> 188 return sort_batch_response_by_response_ids(responses_list)
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/_utils/batching.py:203, in sort_batch_response_by_response_ids(responses)
200 def sort_batch_response_by_response_ids(
201 responses: List["RPCResponse"],
202 ) -> List["RPCResponse"]:
--> 203 if all(response.get("id") is not None for response in responses):
204 # If all responses have an `id`, sort them by `id, since the JSON-RPC 2.0 spec
205 # doesn't guarantee order.
206 return sorted(responses, key=lambda response: response["id"])
207 else:
208 # If any response is missing an `id`, which should only happen on particular
209 # errors, return them in the order they were received and hope that the
210 # provider is returning them in order. Issue a warning.
File ~/.cache/pypoetry/virtualenvs/etherspect-dagster-jFkAMd9f-py3.12/lib/python3.12/site-packages/web3/_utils/batching.py:203, in <genexpr>(.0)
200 def sort_batch_response_by_response_ids(
201 responses: List["RPCResponse"],
202 ) -> List["RPCResponse"]:
--> 203 if all(response.get("id") is not None for response in responses):
204 # If all responses have an `id`, sort them by `id, since the JSON-RPC 2.0 spec
205 # doesn't guarantee order.
206 return sorted(responses, key=lambda response: response["id"])
207 else:
208 # If any response is missing an `id`, which should only happen on particular
209 # errors, return them in the order they were received and hope that the
210 # provider is returning them in order. Issue a warning.
AttributeError: 'str' object has no attribute 'get'Fill this section in if you know how this could or should be fixed
Revise the response parsing behavior in web3/providers/rpc/rpc.py:make_batch_request() and any other relevant files
web3 Version
7.4.0
Python Version
3.12.5
Operating System
linux
Output from pip freeze
aiohappyeyeballs==2.4.3
aiohttp==3.10.10
aiosignal==1.3.1
alembic==1.13.3
annotated-types==0.7.0
anyio==4.6.2.post1
asttokens==2.4.1
attrs==24.2.0
backoff==2.2.1
bitarray==3.0.0
certifi==2024.8.30
charset-normalizer==3.4.0
ckzg==2.0.1
click==8.1.7
coloredlogs==14.0
croniter==3.0.3
cytoolz==1.0.0
dagster==1.8.12
dagster-graphql==1.8.12
dagster-pipes==1.8.12
dagster-webserver==1.8.12
decorator==5.1.1
docstring_parser==0.16
eth-account==0.13.4
eth-hash==0.7.0
eth-keyfile==0.8.1
eth-keys==0.6.0
eth-rlp==2.1.0
eth-typing==5.0.1
eth-utils==5.1.0
eth_abi==5.1.0
executing==2.1.0
filelock==3.16.1
frozenlist==1.5.0
fsspec==2024.10.0
gql==3.5.0
graphene==3.4
graphql-core==3.2.5
graphql-relay==3.2.0
greenlet==3.1.1
grpcio==1.67.0
grpcio-health-checking==1.62.3
h11==0.14.0
hexbytes==1.2.1
httptools==0.6.4
humanfriendly==10.0
idna==3.10
iniconfig==2.0.0
ipdb==0.13.13
ipython==8.28.0
jedi==0.19.1
Jinja2==3.1.4
Mako==1.3.5
markdown-it-py==3.0.0
MarkupSafe==3.0.2
matplotlib-inline==0.1.7
mdurl==0.1.2
more-itertools==10.5.0
multidict==6.1.0
mypy==1.12.1
mypy-extensions==1.0.0
numpy==2.1.2
packaging==24.1
pandas==2.2.3
pandas-stubs==2.2.3.241009
parsimonious==0.10.0
parso==0.8.4
pexpect==4.9.0
pluggy==1.5.0
prompt_toolkit==3.0.48
propcache==0.2.0
protobuf==4.25.5
ptyprocess==0.7.0
pure_eval==0.2.3
pycryptodome==3.21.0
pydantic==2.9.2
pydantic_core==2.23.4
Pygments==2.18.0
pytest==8.3.3
python-dateutil==2.9.0.post0
python-dotenv==1.0.1
pytz==2024.2
pyunormalize==16.0.0
PyYAML==6.0.2
regex==2024.9.11
requests==2.32.3
requests-toolbelt==1.0.0
rich==13.9.2
rlp==4.0.1
setuptools==75.2.0
six==1.16.0
sniffio==1.3.1
SQLAlchemy==2.0.36
stack-data==0.6.3
starlette==0.41.0
structlog==24.4.0
tabulate==0.9.0
tomli==2.0.2
toolz==1.0.0
toposort==1.10
tqdm==4.66.5
traitlets==5.14.3
types-pytz==2024.2.0.20241003
types-requests==2.32.0.20241016
typing_extensions==4.12.2
tzdata==2024.2
universal_pathlib==0.2.5
urllib3==2.2.3
uvicorn==0.32.0
uvloop==0.21.0
watchdog==5.0.3
watchfiles==0.24.0
wcwidth==0.2.13
web3==7.4.0
websockets==13.1
yarl==1.15.5