Skip to content

Commit f9f1172

Browse files
zsfelfoldifjl
andauthored
core/filtermaps: FilterMaps log index generator and search logic (ethereum#31079)
This PR is #1 of a 3-part series that implements the new log index intended to replace core/bloombits. Replaces ethereum#30370 This part implements the new data structure, the log index generator and the search logic. This PR has most of the complexity but it does not affect any existing code yet so maybe it is easier to review separately. FilterMaps data structure explanation: https://gist.github.com/zsfelfoldi/a60795f9da7ae6422f28c7a34e02a07e Log index generator code overview: https://gist.github.com/zsfelfoldi/97105dff0b1a4f5ed557924a24b9b9e7 Search pattern matcher code overview: https://gist.github.com/zsfelfoldi/5981735641c956afb18065e84f8aff34 Note that the possibility of a tree hashing scheme and remote proof protocol are mentioned in the documents above but they are not exactly specified yet. These specs are WIP and will be finalized after the local log indexer/filter code is finalized and merged. --------- Co-authored-by: Felix Lange <[email protected]>
1 parent 78be413 commit f9f1172

17 files changed

+4825
-0
lines changed

core/filtermaps/chain_view.go

+156
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// Copyright 2025 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package filtermaps
18+
19+
import (
20+
"github.com/ethereum/go-ethereum/common"
21+
"github.com/ethereum/go-ethereum/core/types"
22+
"github.com/ethereum/go-ethereum/log"
23+
)
24+
25+
// blockchain represents the underlying blockchain of ChainView.
26+
type blockchain interface {
27+
GetHeader(hash common.Hash, number uint64) *types.Header
28+
GetCanonicalHash(number uint64) common.Hash
29+
GetReceiptsByHash(hash common.Hash) types.Receipts
30+
}
31+
32+
// ChainView represents an immutable view of a chain with a block id and a set
33+
// of receipts associated to each block number and a block hash associated with
34+
// all block numbers except the head block. This is because in the future
35+
// ChainView might represent a view where the head block is currently being
36+
// created. Block id is a unique identifier that can also be calculated for the
37+
// head block.
38+
// Note that the view's head does not have to be the current canonical head
39+
// of the underlying blockchain, it should only possess the block headers
40+
// and receipts up until the expected chain view head.
41+
type ChainView struct {
42+
chain blockchain
43+
headNumber uint64
44+
hashes []common.Hash // block hashes starting backwards from headNumber until first canonical hash
45+
}
46+
47+
// NewChainView creates a new ChainView.
48+
func NewChainView(chain blockchain, number uint64, hash common.Hash) *ChainView {
49+
cv := &ChainView{
50+
chain: chain,
51+
headNumber: number,
52+
hashes: []common.Hash{hash},
53+
}
54+
cv.extendNonCanonical()
55+
return cv
56+
}
57+
58+
// getBlockHash returns the block hash belonging to the given block number.
59+
// Note that the hash of the head block is not returned because ChainView might
60+
// represent a view where the head block is currently being created.
61+
func (cv *ChainView) getBlockHash(number uint64) common.Hash {
62+
if number >= cv.headNumber {
63+
panic("invalid block number")
64+
}
65+
return cv.blockHash(number)
66+
}
67+
68+
// getBlockId returns the unique block id belonging to the given block number.
69+
// Note that it is currently equal to the block hash. In the future it might
70+
// be a different id for future blocks if the log index root becomes part of
71+
// consensus and therefore rendering the index with the new head will happen
72+
// before the hash of that new head is available.
73+
func (cv *ChainView) getBlockId(number uint64) common.Hash {
74+
if number > cv.headNumber {
75+
panic("invalid block number")
76+
}
77+
return cv.blockHash(number)
78+
}
79+
80+
// getReceipts returns the set of receipts belonging to the block at the given
81+
// block number.
82+
func (cv *ChainView) getReceipts(number uint64) types.Receipts {
83+
if number > cv.headNumber {
84+
panic("invalid block number")
85+
}
86+
return cv.chain.GetReceiptsByHash(cv.blockHash(number))
87+
}
88+
89+
// limitedView returns a new chain view that is a truncated version of the parent view.
90+
func (cv *ChainView) limitedView(newHead uint64) *ChainView {
91+
if newHead >= cv.headNumber {
92+
return cv
93+
}
94+
return NewChainView(cv.chain, newHead, cv.blockHash(newHead))
95+
}
96+
97+
// equalViews returns true if the two chain views are equivalent.
98+
func equalViews(cv1, cv2 *ChainView) bool {
99+
if cv1 == nil || cv2 == nil {
100+
return false
101+
}
102+
return cv1.headNumber == cv2.headNumber && cv1.getBlockId(cv1.headNumber) == cv2.getBlockId(cv2.headNumber)
103+
}
104+
105+
// matchViews returns true if the two chain views are equivalent up until the
106+
// specified block number. If the specified number is higher than one of the
107+
// heads then false is returned.
108+
func matchViews(cv1, cv2 *ChainView, number uint64) bool {
109+
if cv1 == nil || cv2 == nil {
110+
return false
111+
}
112+
if cv1.headNumber < number || cv2.headNumber < number {
113+
return false
114+
}
115+
if number == cv1.headNumber || number == cv2.headNumber {
116+
return cv1.getBlockId(number) == cv2.getBlockId(number)
117+
}
118+
return cv1.getBlockHash(number) == cv2.getBlockHash(number)
119+
}
120+
121+
// extendNonCanonical checks whether the previously known reverse list of head
122+
// hashes still ends with one that is canonical on the underlying blockchain.
123+
// If necessary then it traverses further back on the header chain and adds
124+
// more hashes to the list.
125+
func (cv *ChainView) extendNonCanonical() bool {
126+
for {
127+
hash, number := cv.hashes[len(cv.hashes)-1], cv.headNumber-uint64(len(cv.hashes)-1)
128+
if cv.chain.GetCanonicalHash(number) == hash {
129+
return true
130+
}
131+
if number == 0 {
132+
log.Error("Unknown genesis block hash found")
133+
return false
134+
}
135+
header := cv.chain.GetHeader(hash, number)
136+
if header == nil {
137+
log.Error("Header not found", "number", number, "hash", hash)
138+
return false
139+
}
140+
cv.hashes = append(cv.hashes, header.ParentHash)
141+
}
142+
}
143+
144+
// blockHash returns the given block hash without doing the head number check.
145+
func (cv *ChainView) blockHash(number uint64) common.Hash {
146+
if number+uint64(len(cv.hashes)) <= cv.headNumber {
147+
hash := cv.chain.GetCanonicalHash(number)
148+
if !cv.extendNonCanonical() {
149+
return common.Hash{}
150+
}
151+
if number+uint64(len(cv.hashes)) <= cv.headNumber {
152+
return hash
153+
}
154+
}
155+
return cv.hashes[cv.headNumber-number]
156+
}

core/filtermaps/checkpoints.go

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Copyright 2025 The go-ethereum Authors
2+
// This file is part of the go-ethereum library.
3+
//
4+
// The go-ethereum library is free software: you can redistribute it and/or modify
5+
// it under the terms of the GNU Lesser General Public License as published by
6+
// the Free Software Foundation, either version 3 of the License, or
7+
// (at your option) any later version.
8+
//
9+
// The go-ethereum library is distributed in the hope that it will be useful,
10+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
// GNU Lesser General Public License for more details.
13+
//
14+
// You should have received a copy of the GNU Lesser General Public License
15+
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
16+
17+
package filtermaps
18+
19+
import (
20+
_ "embed"
21+
"encoding/json"
22+
23+
"github.com/ethereum/go-ethereum/common"
24+
)
25+
26+
// checkpointList lists checkpoints for finalized epochs of a given chain.
27+
// This allows the indexer to start indexing from the latest available
28+
// checkpoint and then index tail epochs in reverse order.
29+
type checkpointList []epochCheckpoint
30+
31+
// epochCheckpoint specified the last block of the epoch and the first log
32+
// value index where that block starts. This allows a log value iterator to
33+
// be initialized at the epoch boundary.
34+
type epochCheckpoint struct {
35+
BlockNumber uint64 // block that generated the last log value of the given epoch
36+
BlockId common.Hash
37+
FirstIndex uint64 // first log value index of the given block
38+
}
39+
40+
//go:embed checkpoints_mainnet.json
41+
var checkpointsMainnetJSON []byte
42+
43+
//go:embed checkpoints_sepolia.json
44+
var checkpointsSepoliaJSON []byte
45+
46+
//go:embed checkpoints_holesky.json
47+
var checkpointsHoleskyJSON []byte
48+
49+
// checkpoints lists sets of checkpoints for multiple chains. The matching
50+
// checkpoint set is autodetected by the indexer once the canonical chain is
51+
// known.
52+
var checkpoints = []checkpointList{
53+
decodeCheckpoints(checkpointsMainnetJSON),
54+
decodeCheckpoints(checkpointsSepoliaJSON),
55+
decodeCheckpoints(checkpointsHoleskyJSON),
56+
}
57+
58+
func decodeCheckpoints(encoded []byte) (result checkpointList) {
59+
if err := json.Unmarshal(encoded, &result); err != nil {
60+
panic(err)
61+
}
62+
return
63+
}
+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
[
2+
{"blockNumber": 814411, "blockId": "0xf763e96fc3920359c5f706803024b78e83796a3a8563bb5a83c3ddd7cbfde287", "firstIndex": 67107637},
3+
{"blockNumber": 914278, "blockId": "0x0678cf8d53c0d6d27896df657d98cc73bc63ca468b6295068003938ef9b0f927", "firstIndex": 134217671},
4+
{"blockNumber": 1048874, "blockId": "0x3620c3d52a40ff4d9fc58c3104cfa2f327f55592caf6a2394c207a5e00b4f740", "firstIndex": 201326382},
5+
{"blockNumber": 1144441, "blockId": "0x438fb42850f5a0d8e1666de598a4d0106b62da0f7448c62fe029b8cbad35d08d", "firstIndex": 268435440},
6+
{"blockNumber": 1230411, "blockId": "0xf0ee07e60a93910723b259473a253dd9cf674e8b78c4f153b32ad7032efffeeb", "firstIndex": 335543079},
7+
{"blockNumber": 1309112, "blockId": "0xc1646e5ef4b4343880a85b1a4111e3321d609a1225e9cebbe10d1c7abf99e58d", "firstIndex": 402653100},
8+
{"blockNumber": 1380522, "blockId": "0x1617cae91989d97ac6335c4217aa6cc7f7f4c2837e20b3b5211d98d6f9e97e44", "firstIndex": 469761917},
9+
{"blockNumber": 1476962, "blockId": "0xd978455d2618d093dfc685d7f43f61be6dae0fa8a9cb915ae459aa6e0a5525f0", "firstIndex": 536870773},
10+
{"blockNumber": 1533518, "blockId": "0xe7d39d71bd9d5f1f3157c35e0329531a7950a19e3042407e38948b89b5384f78", "firstIndex": 603979664},
11+
{"blockNumber": 1613787, "blockId": "0xa793168d135c075732a618ec367faaed5f359ffa81898c73cb4ec54ec2caa696", "firstIndex": 671088003},
12+
{"blockNumber": 1719099, "blockId": "0xc4394c71a8a24efe64c5ff2afcdd1594f3708524e6084aa7dadd862bd704ab03", "firstIndex": 738196914},
13+
{"blockNumber": 1973165, "blockId": "0xee3a9e959a437c707a3036736ec8d42a9261ac6100972c26f65eedcde315a81d", "firstIndex": 805306333},
14+
{"blockNumber": 2274844, "blockId": "0x76e2d33653ed9282c63ad09d721e1f2e29064aa9c26202e20fc4cc73e8dfe5f6", "firstIndex": 872415141},
15+
{"blockNumber": 2530503, "blockId": "0x59f4e45345f8b8f848be5004fe75c4a28f651864256c3aa9b2da63369432b718", "firstIndex": 939523693},
16+
{"blockNumber": 2781903, "blockId": "0xc981e91c6fb69c5e8146ead738fcfc561831f11d7786d39c7fa533966fc37675", "firstIndex": 1006632906},
17+
{"blockNumber": 3101713, "blockId": "0xc7baa577c91d8439e3fc79002d2113d07ca54a4724bf2f1f5af937b7ba8e1f32", "firstIndex": 1073741382},
18+
{"blockNumber": 3221770, "blockId": "0xa6b8240b7883fcc71aa5001b5ba66c889975c5217e14c16edebdd6f6e23a9424", "firstIndex": 1140850360}
19+
]
20+

0 commit comments

Comments
 (0)