From c2eae4f78f27c5838ed355a07eb47e91df410aee Mon Sep 17 00:00:00 2001 From: NouDaimon Date: Sun, 23 Oct 2022 23:11:38 +0300 Subject: [PATCH 01/59] remove length check to avoid storage read --- contracts/data/IncrementalMerkleTree.sol | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index c9a68ff12..10dc9f743 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -128,7 +128,7 @@ library IncrementalMerkleTree { bytes32 hash ) internal { unchecked { - _set(t.nodes, 0, index, t.height() - 1, hash); + _set(t.nodes, 0, index, t.height() - 1, t.size(), hash); } } @@ -145,6 +145,7 @@ library IncrementalMerkleTree { uint256 rowIndex, uint256 colIndex, uint256 rootIndex, + uint256 rowSize, bytes32 hash ) private { bytes32[] storage row = nodes[rowIndex]; @@ -165,7 +166,7 @@ library IncrementalMerkleTree { mstore(0x20, hash) hash := keccak256(0x00, 0x40) } - } else if (colIndex + 1 < row.length) { + } else if (colIndex + 1 < rowSize) { // sibling is on the right (and sibling exists) assembly { mstore(0x00, row.slot) @@ -178,7 +179,8 @@ library IncrementalMerkleTree { } } - _set(nodes, rowIndex + 1, colIndex >> 1, rootIndex, hash); + rowSize = rowSize % 2 == 0 ? rowSize >> 1 : (rowSize >> 1) + 1; + _set(nodes, rowIndex + 1, colIndex >> 1, rootIndex, rowSize, hash); } } } From 4a359716914eef8d3c819bfdb327608008d132d3 Mon Sep 17 00:00:00 2001 From: NouDaimon Date: Sun, 23 Oct 2022 23:12:39 +0300 Subject: [PATCH 02/59] add natspec --- contracts/data/IncrementalMerkleTree.sol | 1 + 1 file changed, 1 insertion(+) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 10dc9f743..8af51aaad 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -138,6 +138,7 @@ library IncrementalMerkleTree { * @param rowIndex index of current row to update * @param colIndex index of current column to update * @param rootIndex index of root row + * @param rowSize length of row at rowIndex * @param hash hash to store at current position */ function _set( From 5dcf49a21f1d1fec0d74f2c8fbf580095202c1bd Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 15:17:10 -0500 Subject: [PATCH 03/59] store IMT hashes via assembly --- contracts/data/IncrementalMerkleTree.sol | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 8af51aaad..3d3d7cde0 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -151,7 +151,12 @@ library IncrementalMerkleTree { ) private { bytes32[] storage row = nodes[rowIndex]; - row[colIndex] = hash; + // store hash in array via assembly to avoid array length sload + + assembly { + mstore(0x00, row.slot) + sstore(add(keccak256(0x00, 0x20), colIndex), hash) + } if (rowIndex == rootIndex) return; From 4c5850b949ee5b7412a3ddad2cc51b0e23a049b7 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 15:18:18 -0500 Subject: [PATCH 04/59] rename rowSize to rowLength, modify lt comparison for clarity --- contracts/data/IncrementalMerkleTree.sol | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 3d3d7cde0..ea3228578 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -138,7 +138,7 @@ library IncrementalMerkleTree { * @param rowIndex index of current row to update * @param colIndex index of current column to update * @param rootIndex index of root row - * @param rowSize length of row at rowIndex + * @param rowLength length of row at rowIndex * @param hash hash to store at current position */ function _set( @@ -146,7 +146,7 @@ library IncrementalMerkleTree { uint256 rowIndex, uint256 colIndex, uint256 rootIndex, - uint256 rowSize, + uint256 rowLength, bytes32 hash ) private { bytes32[] storage row = nodes[rowIndex]; @@ -172,7 +172,7 @@ library IncrementalMerkleTree { mstore(0x20, hash) hash := keccak256(0x00, 0x40) } - } else if (colIndex + 1 < rowSize) { + } else if (colIndex < rowLength - 1) { // sibling is on the right (and sibling exists) assembly { mstore(0x00, row.slot) @@ -185,8 +185,17 @@ library IncrementalMerkleTree { } } - rowSize = rowSize % 2 == 0 ? rowSize >> 1 : (rowSize >> 1) + 1; - _set(nodes, rowIndex + 1, colIndex >> 1, rootIndex, rowSize, hash); + rowLength = rowLength % 2 == 0 + ? rowLength >> 1 + : (rowLength >> 1) + 1; + _set( + nodes, + rowIndex + 1, + colIndex >> 1, + rootIndex, + rowLength, + hash + ); } } } From af581c571da7d45f97f0d233bc0cdaf787f404b4 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 15:38:44 -0500 Subject: [PATCH 05/59] remove rootIndex variable --- contracts/data/IncrementalMerkleTree.sol | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index ea3228578..39b4fee2c 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -128,7 +128,7 @@ library IncrementalMerkleTree { bytes32 hash ) internal { unchecked { - _set(t.nodes, 0, index, t.height() - 1, t.size(), hash); + _set(t.nodes, 0, index, t.size(), hash); } } @@ -137,7 +137,6 @@ library IncrementalMerkleTree { * @param nodes internal tree structure storage reference * @param rowIndex index of current row to update * @param colIndex index of current column to update - * @param rootIndex index of root row * @param rowLength length of row at rowIndex * @param hash hash to store at current position */ @@ -145,7 +144,6 @@ library IncrementalMerkleTree { bytes32[][] storage nodes, uint256 rowIndex, uint256 colIndex, - uint256 rootIndex, uint256 rowLength, bytes32 hash ) private { @@ -158,7 +156,7 @@ library IncrementalMerkleTree { sstore(add(keccak256(0x00, 0x20), colIndex), hash) } - if (rowIndex == rootIndex) return; + if (rowLength == 1) return; unchecked { if (colIndex & 1 == 1) { @@ -188,14 +186,7 @@ library IncrementalMerkleTree { rowLength = rowLength % 2 == 0 ? rowLength >> 1 : (rowLength >> 1) + 1; - _set( - nodes, - rowIndex + 1, - colIndex >> 1, - rootIndex, - rowLength, - hash - ); + _set(nodes, rowIndex + 1, colIndex >> 1, rowLength, hash); } } } From 045c652eef7ef42cc52bc8c8cedf49dd9bb1ede6 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 15:41:38 -0500 Subject: [PATCH 06/59] optimize rowLength calculation --- contracts/data/IncrementalMerkleTree.sol | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 39b4fee2c..d5cff8240 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -183,10 +183,13 @@ library IncrementalMerkleTree { } } - rowLength = rowLength % 2 == 0 - ? rowLength >> 1 - : (rowLength >> 1) + 1; - _set(nodes, rowIndex + 1, colIndex >> 1, rowLength, hash); + _set( + nodes, + rowIndex + 1, + colIndex >> 1, + (rowLength >> 1) + (rowLength & 1), + hash + ); } } } From dfebf46eb40c8cd49ef7229e5ea4dba19b02399f Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 16:02:27 -0500 Subject: [PATCH 07/59] optimize IMT size function --- contracts/data/IncrementalMerkleTree.sol | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index d5cff8240..a399c2e76 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -15,8 +15,11 @@ library IncrementalMerkleTree { * @return treeSize size of tree */ function size(Tree storage t) internal view returns (uint256 treeSize) { - if (t.height() > 0) { - treeSize = t.nodes[0].length; + bytes32[][] storage nodes = t.nodes; + + assembly { + mstore(0x00, nodes.slot) + treeSize := sload(keccak256(0x00, 0x20)) } } From cc90a1dea655bee8aed9d0942e80f3d3d53f453b Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 19:34:20 -0500 Subject: [PATCH 08/59] wrap contents of IMT pop function in unchecked block --- contracts/data/IncrementalMerkleTree.sol | 38 +++++++++++++----------- test/data/IncrementalMerkleTree.ts | 2 +- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index a399c2e76..8a6f25507 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -91,31 +91,33 @@ library IncrementalMerkleTree { } function pop(Tree storage t) internal { - uint256 treeHeight = t.height(); - uint256 treeSize = t.size() - 1; + unchecked { + uint256 treeHeight = t.height(); + uint256 treeSize = t.size() - 1; - // remove layer if tree has excess capacity + // remove layer if tree has excess capacity - if (treeSize == (1 << treeHeight) >> 2) { - treeHeight--; - t.nodes.pop(); - } + if (treeSize == (1 << treeHeight) >> 2) { + treeHeight--; + t.nodes.pop(); + } - // remove columns if rows are too long + // remove columns if rows are too long - uint256 row; - uint256 col = treeSize; + uint256 row; + uint256 col = treeSize; - while (row < treeHeight && t.nodes[row].length > col) { - t.nodes[row].pop(); - row++; - col = (col + 1) >> 1; - } + while (row < treeHeight && t.nodes[row].length > col) { + t.nodes[row].pop(); + row++; + col = (col + 1) >> 1; + } - // recalculate hashes + // recalculate hashes - if (treeSize > 0) { - t.set(treeSize - 1, t.at(treeSize - 1)); + if (treeSize > 0) { + t.set(treeSize - 1, t.at(treeSize - 1)); + } } } diff --git a/test/data/IncrementalMerkleTree.ts b/test/data/IncrementalMerkleTree.ts index c4f3c98a5..ca05afa7c 100644 --- a/test/data/IncrementalMerkleTree.ts +++ b/test/data/IncrementalMerkleTree.ts @@ -186,7 +186,7 @@ describe('IncrementalMerkleTree', function () { describe('reverts if', () => { it('tree is size zero', async () => { await expect(instance.pop()).to.be.revertedWithPanic( - PANIC_CODES.ARITHMETIC_UNDER_OR_OVERFLOW, + PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, ); }); }); From ba217cd2f80662b7e11a9099932348da8745d501 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 21:01:54 -0500 Subject: [PATCH 09/59] remove unused unchecked block --- contracts/data/IncrementalMerkleTree.sol | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 8a6f25507..14f2e2eac 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -132,9 +132,7 @@ library IncrementalMerkleTree { uint256 index, bytes32 hash ) internal { - unchecked { - _set(t.nodes, 0, index, t.size(), hash); - } + _set(t.nodes, 0, index, t.size(), hash); } /** From 3fdae957d6a06ca6eec657b0386f47aa2e55ad0e Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 21:05:45 -0500 Subject: [PATCH 10/59] do not recalculate hashes if layer is removed on pop --- contracts/data/IncrementalMerkleTree.sol | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 14f2e2eac..43480f355 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -95,13 +95,6 @@ library IncrementalMerkleTree { uint256 treeHeight = t.height(); uint256 treeSize = t.size() - 1; - // remove layer if tree has excess capacity - - if (treeSize == (1 << treeHeight) >> 2) { - treeHeight--; - t.nodes.pop(); - } - // remove columns if rows are too long uint256 row; @@ -113,9 +106,12 @@ library IncrementalMerkleTree { col = (col + 1) >> 1; } - // recalculate hashes + // if new tree is full, remove excess layer + // if no layer is removed, recalculate hashes - if (treeSize > 0) { + if (treeSize == ((1 << treeHeight) >> 2)) { + t.nodes.pop(); + } else { t.set(treeSize - 1, t.at(treeSize - 1)); } } From bff831ed910b8265184fc1c5d440a9631ed7d8f4 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 22:35:48 -0500 Subject: [PATCH 11/59] emphasize prallel structure of push and pop functions --- contracts/data/IncrementalMerkleTree.sol | 29 ++++++++++++------------ 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 43480f355..8766ea5c8 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -63,56 +63,57 @@ library IncrementalMerkleTree { */ function push(Tree storage t, bytes32 hash) internal { unchecked { - uint256 treeHeight = t.height(); - uint256 treeSize = t.size(); + // index to add to tree + uint256 updateIndex = t.size(); // add new layer if tree is at capacity - if (treeSize == (1 << treeHeight) >> 1) { + if (updateIndex == (1 << t.height()) >> 1) { t.nodes.push(); - treeHeight++; } // add new columns if rows are full uint256 row; - uint256 col = treeSize; + uint256 col = updateIndex; - while (row < treeHeight && t.nodes[row].length <= col) { + while (col == t.nodes[row].length) { t.nodes[row].push(); row++; + if (col == 0) break; col >>= 1; } // add hash to tree - t.set(treeSize, hash); + t.set(updateIndex, hash); } } function pop(Tree storage t) internal { unchecked { - uint256 treeHeight = t.height(); - uint256 treeSize = t.size() - 1; + // index to remove from tree + uint256 updateIndex = t.size() - 1; // remove columns if rows are too long uint256 row; - uint256 col = treeSize; + uint256 col = updateIndex; - while (row < treeHeight && t.nodes[row].length > col) { + while (col < t.nodes[row].length) { t.nodes[row].pop(); row++; - col = (col + 1) >> 1; + col >>= 1; + if (col == 0) break; } // if new tree is full, remove excess layer // if no layer is removed, recalculate hashes - if (treeSize == ((1 << treeHeight) >> 2)) { + if (updateIndex == (1 << t.height()) >> 2) { t.nodes.pop(); } else { - t.set(treeSize - 1, t.at(treeSize - 1)); + t.set(updateIndex - 1, t.at(updateIndex - 1)); } } } From 761ce875ebabaccd0074c4f12f12c3e0e0813d04 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 22:37:00 -0500 Subject: [PATCH 12/59] optimize recursive row length calculation --- contracts/data/IncrementalMerkleTree.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 8766ea5c8..ca0220721 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -187,7 +187,7 @@ library IncrementalMerkleTree { nodes, rowIndex + 1, colIndex >> 1, - (rowLength >> 1) + (rowLength & 1), + (rowLength + 1) >> 1, hash ); } From 24b483431ec061dad34578620fb115b03ad96140 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sun, 23 Oct 2022 23:08:55 -0500 Subject: [PATCH 13/59] replace lt with not eq comparison --- contracts/data/IncrementalMerkleTree.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index ca0220721..1db21e67d 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -100,7 +100,7 @@ library IncrementalMerkleTree { uint256 row; uint256 col = updateIndex; - while (col < t.nodes[row].length) { + while (col != t.nodes[row].length) { t.nodes[row].pop(); row++; col >>= 1; From 886ba4b4c6794842afdc006ecdee6244bd358db2 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Mon, 24 Oct 2022 00:43:51 -0500 Subject: [PATCH 14/59] add additional at function revert test --- test/data/IncrementalMerkleTree.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/data/IncrementalMerkleTree.ts b/test/data/IncrementalMerkleTree.ts index ca05afa7c..fd894783c 100644 --- a/test/data/IncrementalMerkleTree.ts +++ b/test/data/IncrementalMerkleTree.ts @@ -134,11 +134,19 @@ describe('IncrementalMerkleTree', function () { }); describe('reverts if', () => { - it('index is out of bounds', async () => { + it('tree is size zero', async () => { await expect(instance.callStatic.at(0)).to.be.revertedWithPanic( PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, ); }); + + it('index is out of bounds', async () => { + await instance.push(randomHash()); + + await expect(instance.callStatic.at(1)).to.be.revertedWithPanic( + PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, + ); + }); }); }); From e3116c30fb1e31c6f9af9f3289fee45c94dca265 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Mon, 24 Oct 2022 01:00:26 -0500 Subject: [PATCH 15/59] use assembly for height and root functions --- contracts/data/IncrementalMerkleTree.sol | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 1db21e67d..d8837aba5 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -27,10 +27,14 @@ library IncrementalMerkleTree { * @notice query one-indexed height of tree * @dev conventional zero-indexed height would require the use of signed integers, so height is one-indexed instead * @param t Tree struct storage reference - * @return one-indexed height of tree + * @return treeHeight one-indexed height of tree */ - function height(Tree storage t) internal view returns (uint256) { - return t.nodes.length; + function height(Tree storage t) internal view returns (uint256 treeHeight) { + bytes32[][] storage nodes = t.nodes; + + assembly { + treeHeight := sload(nodes.slot) + } } /** @@ -39,11 +43,15 @@ library IncrementalMerkleTree { * @return hash root hash */ function root(Tree storage t) internal view returns (bytes32 hash) { + bytes32[][] storage nodes = t.nodes; + uint256 treeHeight = t.height(); if (treeHeight > 0) { - unchecked { - hash = t.nodes[treeHeight - 1][0]; + assembly { + mstore(0x00, nodes.slot) + mstore(0x00, add(keccak256(0x00, 0x20), sub(treeHeight, 1))) + hash := sload(keccak256(0x00, 0x20)) } } } From 511153cf3e77bcc15de219bc848b1ab1146d5b6f Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Mon, 24 Oct 2022 02:00:53 -0500 Subject: [PATCH 16/59] circumvent all Solididity safety features for underlying IMT arrays --- contracts/data/IncrementalMerkleTree.sol | 115 +++++++++++++---------- 1 file changed, 63 insertions(+), 52 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index d8837aba5..477a2f708 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -6,7 +6,7 @@ library IncrementalMerkleTree { using IncrementalMerkleTree for Tree; struct Tree { - bytes32[][] nodes; + bytes32[][] __nodes; } /** @@ -15,10 +15,8 @@ library IncrementalMerkleTree { * @return treeSize size of tree */ function size(Tree storage t) internal view returns (uint256 treeSize) { - bytes32[][] storage nodes = t.nodes; - assembly { - mstore(0x00, nodes.slot) + mstore(0x00, t.slot) treeSize := sload(keccak256(0x00, 0x20)) } } @@ -30,10 +28,8 @@ library IncrementalMerkleTree { * @return treeHeight one-indexed height of tree */ function height(Tree storage t) internal view returns (uint256 treeHeight) { - bytes32[][] storage nodes = t.nodes; - assembly { - treeHeight := sload(nodes.slot) + treeHeight := sload(t.slot) } } @@ -43,13 +39,11 @@ library IncrementalMerkleTree { * @return hash root hash */ function root(Tree storage t) internal view returns (bytes32 hash) { - bytes32[][] storage nodes = t.nodes; - uint256 treeHeight = t.height(); if (treeHeight > 0) { assembly { - mstore(0x00, nodes.slot) + mstore(0x00, t.slot) mstore(0x00, add(keccak256(0x00, 0x20), sub(treeHeight, 1))) hash := sload(keccak256(0x00, 0x20)) } @@ -61,7 +55,15 @@ library IncrementalMerkleTree { view returns (bytes32 hash) { - hash = t.nodes[0][index]; + if (index >= t.size()) { + new bytes32[](0)[1]; + } + + assembly { + mstore(0x00, t.slot) + mstore(0x00, keccak256(0x00, 0x20)) + hash := sload(add(keccak256(0x00, 0x20), index)) + } } /** @@ -70,56 +72,60 @@ library IncrementalMerkleTree { * @param hash to add */ function push(Tree storage t, bytes32 hash) internal { - unchecked { - // index to add to tree - uint256 updateIndex = t.size(); + // index to add to tree + uint256 updateIndex = t.size(); - // add new layer if tree is at capacity + // update stored tree size - if (updateIndex == (1 << t.height()) >> 1) { - t.nodes.push(); - } + assembly { + mstore(0x00, t.slot) + sstore(keccak256(0x00, 0x20), add(updateIndex, 1)) + } - // add new columns if rows are full + // add new layer if tree is at capacity - uint256 row; - uint256 col = updateIndex; + uint256 treeHeight = t.height(); - while (col == t.nodes[row].length) { - t.nodes[row].push(); - row++; - if (col == 0) break; - col >>= 1; + if (updateIndex == (1 << treeHeight) >> 1) { + // increment tree height in storage + assembly { + sstore(t.slot, add(treeHeight, 1)) } + } - // add hash to tree + // add hash to tree - t.set(updateIndex, hash); - } + t.set(updateIndex, hash); } function pop(Tree storage t) internal { + uint256 treeSize = t.size(); + + if (treeSize == 0) { + new bytes32[](0)[1]; + } + unchecked { // index to remove from tree - uint256 updateIndex = t.size() - 1; + uint256 updateIndex = treeSize - 1; - // remove columns if rows are too long + // update stored tree size - uint256 row; - uint256 col = updateIndex; - - while (col != t.nodes[row].length) { - t.nodes[row].pop(); - row++; - col >>= 1; - if (col == 0) break; + assembly { + mstore(0x00, t.slot) + sstore(keccak256(0x00, 0x20), updateIndex) } // if new tree is full, remove excess layer // if no layer is removed, recalculate hashes - if (updateIndex == (1 << t.height()) >> 2) { - t.nodes.pop(); + uint256 treeHeight = t.height(); + + if (updateIndex == (1 << treeHeight) >> 2) { + // decrement tree height in storage + assembly { + sstore(t.slot, sub(treeHeight, 1)) + } } else { t.set(updateIndex - 1, t.at(updateIndex - 1)); } @@ -137,25 +143,36 @@ library IncrementalMerkleTree { uint256 index, bytes32 hash ) internal { - _set(t.nodes, 0, index, t.size(), hash); + uint256 treeSize = t.size(); + + if (index >= treeSize) { + new bytes32[](0)[1]; + } + + _set(t, 0, index, treeSize, hash); } /** * @notice update element in tree and recursively recalculate hashes - * @param nodes internal tree structure storage reference + * @param t Tree struct storage reference * @param rowIndex index of current row to update * @param colIndex index of current column to update * @param rowLength length of row at rowIndex * @param hash hash to store at current position */ function _set( - bytes32[][] storage nodes, + Tree storage t, uint256 rowIndex, uint256 colIndex, uint256 rowLength, bytes32 hash ) private { - bytes32[] storage row = nodes[rowIndex]; + bytes32[] storage row; + + assembly { + mstore(0x00, t.slot) + row.slot := add(keccak256(0x00, 0x20), rowIndex) + } // store hash in array via assembly to avoid array length sload @@ -191,13 +208,7 @@ library IncrementalMerkleTree { } } - _set( - nodes, - rowIndex + 1, - colIndex >> 1, - (rowLength + 1) >> 1, - hash - ); + _set(t, rowIndex + 1, colIndex >> 1, (rowLength + 1) >> 1, hash); } } } From ad3f47d5aaa09d06dfd3d6014c7d84192924a89e Mon Sep 17 00:00:00 2001 From: NouDaimon Date: Mon, 24 Oct 2022 13:03:44 +0300 Subject: [PATCH 17/59] wrap root functionality in assembly block --- contracts/data/IncrementalMerkleTree.sol | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 477a2f708..aaae104bb 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -39,10 +39,9 @@ library IncrementalMerkleTree { * @return hash root hash */ function root(Tree storage t) internal view returns (bytes32 hash) { - uint256 treeHeight = t.height(); - - if (treeHeight > 0) { - assembly { + assembly { + let treeHeight := sload(t.slot) + if gt(treeHeight, 0) { mstore(0x00, t.slot) mstore(0x00, add(keccak256(0x00, 0x20), sub(treeHeight, 1))) hash := sload(keccak256(0x00, 0x20)) From d455bb90f7d1a49ce9adebee91a084718f59f011 Mon Sep 17 00:00:00 2001 From: NouDaimon Date: Mon, 24 Oct 2022 13:52:34 +0300 Subject: [PATCH 18/59] remove unrequired mstore operation --- contracts/data/IncrementalMerkleTree.sol | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index aaae104bb..1fccf35f6 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -186,7 +186,6 @@ library IncrementalMerkleTree { if (colIndex & 1 == 1) { // sibling is on the left assembly { - mstore(0x00, row.slot) let sibling := sload( add(keccak256(0x00, 0x20), sub(colIndex, 1)) ) @@ -197,7 +196,6 @@ library IncrementalMerkleTree { } else if (colIndex < rowLength - 1) { // sibling is on the right (and sibling exists) assembly { - mstore(0x00, row.slot) let sibling := sload( add(keccak256(0x00, 0x20), add(colIndex, 1)) ) @@ -206,8 +204,8 @@ library IncrementalMerkleTree { hash := keccak256(0x00, 0x40) } } - - _set(t, rowIndex + 1, colIndex >> 1, (rowLength + 1) >> 1, hash); } + + _set(t, rowIndex + 1, colIndex >> 1, (rowLength + 1) >> 1, hash); } } From 1dedd3bb01466789a32aed7b00226e9413c7acaa Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 22 Mar 2025 01:13:11 -0600 Subject: [PATCH 19/59] add comments explaining some IMT assembly blocks --- contracts/data/IncrementalMerkleTree.sol | 27 ++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 288bf389b..86c861d2c 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -16,6 +16,10 @@ library IncrementalMerkleTree { */ function size(Tree storage t) internal view returns (uint256 treeSize) { assembly { + // assembly block equivalent to: + // + // if (t.height() > 0) treeSize = t.__nodes[0].length; + mstore(0x00, t.slot) treeSize := sload(keccak256(0x00, 0x20)) } @@ -29,6 +33,10 @@ library IncrementalMerkleTree { */ function height(Tree storage t) internal view returns (uint256 treeHeight) { assembly { + // assembly block equivalent to: + // + // treeHeight = t.__nodes.length; + treeHeight := sload(t.slot) } } @@ -40,6 +48,10 @@ library IncrementalMerkleTree { */ function root(Tree storage t) internal view returns (bytes32 hash) { assembly { + // assembly block equivalent to: + // + // if (t.height() > 0) hash = t.__nodes[t.height() - 1][0]; + let treeHeight := sload(t.slot) if gt(treeHeight, 0) { mstore(0x00, t.slot) @@ -54,10 +66,15 @@ library IncrementalMerkleTree { uint256 index ) internal view returns (bytes32 hash) { if (index >= t.size()) { + // force a panic of the same type as array out-of-bounds access new bytes32[](0)[1]; } assembly { + // assembly block equivalent to: + // + // hash = t.__nodes[0][index]; + mstore(0x00, t.slot) mstore(0x00, keccak256(0x00, 0x20)) hash := sload(add(keccak256(0x00, 0x20), index)) @@ -100,6 +117,7 @@ library IncrementalMerkleTree { uint256 treeSize = t.size(); if (treeSize == 0) { + // force a panic of the same type as array out-of-bounds access new bytes32[](0)[1]; } @@ -140,6 +158,7 @@ library IncrementalMerkleTree { uint256 treeSize = t.size(); if (index >= treeSize) { + // force a panic of the same type as array out-of-bounds access new bytes32[](0)[1]; } @@ -164,6 +183,10 @@ library IncrementalMerkleTree { bytes32[] storage row; assembly { + // assembly block equivalent to: + // + // row = nodes[rowIndex]; + mstore(0x00, t.slot) row.slot := add(keccak256(0x00, 0x20), rowIndex) } @@ -171,6 +194,10 @@ library IncrementalMerkleTree { // store hash in array via assembly to avoid array length sload assembly { + // assembly block equivalent to: + // + // row[colIndex] = hash; + mstore(0x00, row.slot) sstore(add(keccak256(0x00, 0x20), colIndex), hash) } From fbc511516091a5e0348fc4f6ca28a7255cfc2939 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 22 Mar 2025 01:19:19 -0600 Subject: [PATCH 20/59] panic via assembly rather than inline invalid array indexing --- contracts/data/IncrementalMerkleTree.sol | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 86c861d2c..252f175bb 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -66,8 +66,11 @@ library IncrementalMerkleTree { uint256 index ) internal view returns (bytes32 hash) { if (index >= t.size()) { - // force a panic of the same type as array out-of-bounds access - new bytes32[](0)[1]; + assembly { + mstore(0x00, 0x4e487b71) + mstore(0x20, 0x32) + revert(0x1c, 0x24) + } } assembly { @@ -117,8 +120,11 @@ library IncrementalMerkleTree { uint256 treeSize = t.size(); if (treeSize == 0) { - // force a panic of the same type as array out-of-bounds access - new bytes32[](0)[1]; + assembly { + mstore(0x00, 0x4e487b71) + mstore(0x20, 0x32) + revert(0x1c, 0x24) + } } unchecked { @@ -158,8 +164,11 @@ library IncrementalMerkleTree { uint256 treeSize = t.size(); if (index >= treeSize) { - // force a panic of the same type as array out-of-bounds access - new bytes32[](0)[1]; + assembly { + mstore(0x00, 0x4e487b71) + mstore(0x20, 0x32) + revert(0x1c, 0x24) + } } _set(t, 0, index, treeSize, hash); From f90537797f3624801096345031b0e958c923422a Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Thu, 27 Mar 2025 20:07:43 -0600 Subject: [PATCH 21/59] combine adjacent assembly blocks --- contracts/data/IncrementalMerkleTree.sol | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 7c1bfe859..2c641b200 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -191,21 +191,16 @@ library IncrementalMerkleTree { ) private { bytes32[] storage row; + // store hash in array via assembly to avoid array length sload + assembly { // assembly block equivalent to: // // row = nodes[rowIndex]; + // row[colIndex] = hash; mstore(0x00, t.slot) row.slot := add(keccak256(0x00, 0x20), rowIndex) - } - - // store hash in array via assembly to avoid array length sload - - assembly { - // assembly block equivalent to: - // - // row[colIndex] = hash; mstore(0x00, row.slot) sstore(add(keccak256(0x00, 0x20), colIndex), hash) From f831925b90b21ebfed9705fe76a338eacc376772 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Thu, 27 Mar 2025 20:10:25 -0600 Subject: [PATCH 22/59] remove unused storage reference --- contracts/data/IncrementalMerkleTree.sol | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 2c641b200..4bef6b0cd 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -189,20 +189,16 @@ library IncrementalMerkleTree { uint256 rowLength, bytes32 hash ) private { - bytes32[] storage row; - // store hash in array via assembly to avoid array length sload assembly { // assembly block equivalent to: // - // row = nodes[rowIndex]; + // bytes32[] storage row = nodes[rowIndex]; // row[colIndex] = hash; mstore(0x00, t.slot) - row.slot := add(keccak256(0x00, 0x20), rowIndex) - - mstore(0x00, row.slot) + mstore(0x00, add(keccak256(0x00, 0x20), rowIndex)) sstore(add(keccak256(0x00, 0x20), colIndex), hash) } From d44b6e0de82302578bd85c1032a0c57ea611829d Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Thu, 27 Mar 2025 20:34:46 -0600 Subject: [PATCH 23/59] rename t to self within methods --- contracts/data/IncrementalMerkleTree.sol | 78 ++++++++++++------------ 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 4bef6b0cd..0ca0c49af 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -11,16 +11,16 @@ library IncrementalMerkleTree { /** * @notice query number of elements contained in tree - * @param t Tree struct storage reference + * @param self Tree struct storage reference * @return treeSize size of tree */ - function size(Tree storage t) internal view returns (uint256 treeSize) { + function size(Tree storage self) internal view returns (uint256 treeSize) { assembly { // assembly block equivalent to: // - // if (t.height() > 0) treeSize = t.__nodes[0].length; + // if (self.height() > 0) treeSize = self.__nodes[0].length; - mstore(0x00, t.slot) + mstore(0x00, self.slot) treeSize := sload(keccak256(0x00, 0x20)) } } @@ -28,33 +28,35 @@ library IncrementalMerkleTree { /** * @notice query one-indexed height of tree * @dev conventional zero-indexed height would require the use of signed integers, so height is one-indexed instead - * @param t Tree struct storage reference + * @param self Tree struct storage reference * @return treeHeight one-indexed height of tree */ - function height(Tree storage t) internal view returns (uint256 treeHeight) { + function height( + Tree storage self + ) internal view returns (uint256 treeHeight) { assembly { // assembly block equivalent to: // - // treeHeight = t.__nodes.length; + // treeHeight = self.__nodes.length; - treeHeight := sload(t.slot) + treeHeight := sload(self.slot) } } /** * @notice query Merkle root - * @param t Tree struct storage reference + * @param self Tree struct storage reference * @return hash root hash */ - function root(Tree storage t) internal view returns (bytes32 hash) { + function root(Tree storage self) internal view returns (bytes32 hash) { assembly { // assembly block equivalent to: // - // if (t.height() > 0) hash = t.__nodes[t.height() - 1][0]; + // if (self.height() > 0) hash = self.__nodes[self.height() - 1][0]; - let treeHeight := sload(t.slot) + let treeHeight := sload(self.slot) if gt(treeHeight, 0) { - mstore(0x00, t.slot) + mstore(0x00, self.slot) mstore(0x00, add(keccak256(0x00, 0x20), sub(treeHeight, 1))) hash := sload(keccak256(0x00, 0x20)) } @@ -62,10 +64,10 @@ library IncrementalMerkleTree { } function at( - Tree storage t, + Tree storage self, uint256 index ) internal view returns (bytes32 hash) { - if (index >= t.size()) { + if (index >= self.size()) { assembly { mstore(0x00, 0x4e487b71) mstore(0x20, 0x32) @@ -76,9 +78,9 @@ library IncrementalMerkleTree { assembly { // assembly block equivalent to: // - // hash = t.__nodes[0][index]; + // hash = self.__nodes[0][index]; - mstore(0x00, t.slot) + mstore(0x00, self.slot) mstore(0x00, keccak256(0x00, 0x20)) hash := sload(add(keccak256(0x00, 0x20), index)) } @@ -86,38 +88,38 @@ library IncrementalMerkleTree { /** * @notice add new element to tree - * @param t Tree struct storage reference + * @param self Tree struct storage reference * @param hash to add */ - function push(Tree storage t, bytes32 hash) internal { + function push(Tree storage self, bytes32 hash) internal { // index to add to tree - uint256 updateIndex = t.size(); + uint256 updateIndex = self.size(); // update stored tree size assembly { - mstore(0x00, t.slot) + mstore(0x00, self.slot) sstore(keccak256(0x00, 0x20), add(updateIndex, 1)) } // add new layer if tree is at capacity - uint256 treeHeight = t.height(); + uint256 treeHeight = self.height(); if (updateIndex == (1 << treeHeight) >> 1) { // increment tree height in storage assembly { - sstore(t.slot, add(treeHeight, 1)) + sstore(self.slot, add(treeHeight, 1)) } } // add hash to tree - t.set(updateIndex, hash); + self.set(updateIndex, hash); } - function pop(Tree storage t) internal { - uint256 treeSize = t.size(); + function pop(Tree storage self) internal { + uint256 treeSize = self.size(); if (treeSize == 0) { assembly { @@ -134,34 +136,34 @@ library IncrementalMerkleTree { // update stored tree size assembly { - mstore(0x00, t.slot) + mstore(0x00, self.slot) sstore(keccak256(0x00, 0x20), updateIndex) } // if new tree is full, remove excess layer // if no layer is removed, recalculate hashes - uint256 treeHeight = t.height(); + uint256 treeHeight = self.height(); if (updateIndex == (1 << treeHeight) >> 2) { // decrement tree height in storage assembly { - sstore(t.slot, sub(treeHeight, 1)) + sstore(self.slot, sub(treeHeight, 1)) } } else { - t.set(updateIndex - 1, t.at(updateIndex - 1)); + self.set(updateIndex - 1, self.at(updateIndex - 1)); } } } /** * @notice update existing element in tree - * @param t Tree struct storage reference + * @param self Tree struct storage reference * @param index index to update * @param hash new hash to add */ - function set(Tree storage t, uint256 index, bytes32 hash) internal { - uint256 treeSize = t.size(); + function set(Tree storage self, uint256 index, bytes32 hash) internal { + uint256 treeSize = self.size(); if (index >= treeSize) { assembly { @@ -171,19 +173,19 @@ library IncrementalMerkleTree { } } - _set(t, 0, index, treeSize, hash); + _set(self, 0, index, treeSize, hash); } /** * @notice update element in tree and recursively recalculate hashes - * @param t Tree struct storage reference + * @param self Tree struct storage reference * @param rowIndex index of current row to update * @param colIndex index of current column to update * @param rowLength length of row at rowIndex * @param hash hash to store at current position */ function _set( - Tree storage t, + Tree storage self, uint256 rowIndex, uint256 colIndex, uint256 rowLength, @@ -197,7 +199,7 @@ library IncrementalMerkleTree { // bytes32[] storage row = nodes[rowIndex]; // row[colIndex] = hash; - mstore(0x00, t.slot) + mstore(0x00, self.slot) mstore(0x00, add(keccak256(0x00, 0x20), rowIndex)) sstore(add(keccak256(0x00, 0x20), colIndex), hash) } @@ -228,6 +230,6 @@ library IncrementalMerkleTree { } } - _set(t, rowIndex + 1, colIndex >> 1, (rowLength + 1) >> 1, hash); + _set(self, rowIndex + 1, colIndex >> 1, (rowLength + 1) >> 1, hash); } } From 6b3022796cf7543c81b61704c1e1faca1801d9a9 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 13:45:01 -0600 Subject: [PATCH 24/59] draft rewrite IncrementalMerkleTree --- contracts/data/IncrementalMerkleTree.sol | 246 ++++++----------------- test/data/IncrementalMerkleTree.ts | 61 +++--- 2 files changed, 100 insertions(+), 207 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 0ca0c49af..37c3f9a0d 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -6,230 +6,110 @@ library IncrementalMerkleTree { using IncrementalMerkleTree for Tree; struct Tree { - bytes32[][] __nodes; + // array always has odd length + // elements are stored at even indexes + bytes32[] _elements; } - /** - * @notice query number of elements contained in tree - * @param self Tree struct storage reference - * @return treeSize size of tree - */ - function size(Tree storage self) internal view returns (uint256 treeSize) { - assembly { - // assembly block equivalent to: - // - // if (self.height() > 0) treeSize = self.__nodes[0].length; + function elements( + Tree storage self + ) internal view returns (bytes32[] memory els) { + return self._elements; + } - mstore(0x00, self.slot) - treeSize := sload(keccak256(0x00, 0x20)) - } + function size(Tree storage self) internal view returns (uint256 treeSize) { + treeSize = (self._elements.length + 1) >> 1; } - /** - * @notice query one-indexed height of tree - * @dev conventional zero-indexed height would require the use of signed integers, so height is one-indexed instead - * @param self Tree struct storage reference - * @return treeHeight one-indexed height of tree - */ function height( Tree storage self ) internal view returns (uint256 treeHeight) { - assembly { - // assembly block equivalent to: - // - // treeHeight = self.__nodes.length; + uint256 treeSize = self.size(); + + if (treeSize == 0) revert(); - treeHeight := sload(self.slot) + while (1 << treeHeight < treeSize) { + treeHeight++; } } - /** - * @notice query Merkle root - * @param self Tree struct storage reference - * @return hash root hash - */ - function root(Tree storage self) internal view returns (bytes32 hash) { - assembly { - // assembly block equivalent to: - // - // if (self.height() > 0) hash = self.__nodes[self.height() - 1][0]; - - let treeHeight := sload(self.slot) - if gt(treeHeight, 0) { - mstore(0x00, self.slot) - mstore(0x00, add(keccak256(0x00, 0x20), sub(treeHeight, 1))) - hash := sload(keccak256(0x00, 0x20)) - } + function root(Tree storage self) internal view returns (bytes32 rootHash) { + unchecked { + return self._elements[(1 << self.height()) - 1]; } } function at( Tree storage self, uint256 index - ) internal view returns (bytes32 hash) { - if (index >= self.size()) { - assembly { - mstore(0x00, 0x4e487b71) - mstore(0x20, 0x32) - revert(0x1c, 0x24) - } - } - - assembly { - // assembly block equivalent to: - // - // hash = self.__nodes[0][index]; - - mstore(0x00, self.slot) - mstore(0x00, keccak256(0x00, 0x20)) - hash := sload(add(keccak256(0x00, 0x20), index)) - } + ) internal view returns (bytes32 element) { + element = self._elements[index << 1]; } - /** - * @notice add new element to tree - * @param self Tree struct storage reference - * @param hash to add - */ - function push(Tree storage self, bytes32 hash) internal { - // index to add to tree - uint256 updateIndex = self.size(); - - // update stored tree size + function push(Tree storage self, bytes32 element) internal { + uint256 treeSize = self.size() + 1; + uint256 len = (treeSize << 1) - 1; assembly { - mstore(0x00, self.slot) - sstore(keccak256(0x00, 0x20), add(updateIndex, 1)) + sstore(self.slot, len) } - // add new layer if tree is at capacity - - uint256 treeHeight = self.height(); - - if (updateIndex == (1 << treeHeight) >> 1) { - // increment tree height in storage - assembly { - sstore(self.slot, add(treeHeight, 1)) - } - } - - // add hash to tree - - self.set(updateIndex, hash); + _set(self, 0, (treeSize - 1) << 1, element, len); } function pop(Tree storage self) internal { - uint256 treeSize = self.size(); + uint256 treeSize = self.size() - 1; + uint256 len = treeSize == 0 ? 0 : (treeSize << 1) - 1; - if (treeSize == 0) { - assembly { - mstore(0x00, 0x4e487b71) - mstore(0x20, 0x32) - revert(0x1c, 0x24) - } + assembly { + sstore(self.slot, len) } - unchecked { - // index to remove from tree - uint256 updateIndex = treeSize - 1; - - // update stored tree size - - assembly { - mstore(0x00, self.slot) - sstore(keccak256(0x00, 0x20), updateIndex) - } - - // if new tree is full, remove excess layer - // if no layer is removed, recalculate hashes + if (treeSize == 0) return; - uint256 treeHeight = self.height(); - - if (updateIndex == (1 << treeHeight) >> 2) { - // decrement tree height in storage - assembly { - sstore(self.slot, sub(treeHeight, 1)) - } - } else { - self.set(updateIndex - 1, self.at(updateIndex - 1)); - } - } + _set(self, 0, (treeSize - 1) << 1, self.at(treeSize - 1), len); } - /** - * @notice update existing element in tree - * @param self Tree struct storage reference - * @param index index to update - * @param hash new hash to add - */ - function set(Tree storage self, uint256 index, bytes32 hash) internal { - uint256 treeSize = self.size(); - - if (index >= treeSize) { - assembly { - mstore(0x00, 0x4e487b71) - mstore(0x20, 0x32) - revert(0x1c, 0x24) - } - } - - _set(self, 0, index, treeSize, hash); + function set(Tree storage self, uint256 index, bytes32 element) internal { + _set(self, 0, index << 1, element, self._elements.length); } - /** - * @notice update element in tree and recursively recalculate hashes - * @param self Tree struct storage reference - * @param rowIndex index of current row to update - * @param colIndex index of current column to update - * @param rowLength length of row at rowIndex - * @param hash hash to store at current position - */ function _set( Tree storage self, - uint256 rowIndex, - uint256 colIndex, - uint256 rowLength, - bytes32 hash - ) private { - // store hash in array via assembly to avoid array length sload - - assembly { - // assembly block equivalent to: - // - // bytes32[] storage row = nodes[rowIndex]; - // row[colIndex] = hash; - - mstore(0x00, self.slot) - mstore(0x00, add(keccak256(0x00, 0x20), rowIndex)) - sstore(add(keccak256(0x00, 0x20), colIndex), hash) + uint256 depth, + uint256 index, + bytes32 element, + uint256 len + ) internal { + if (index < len) { + // write element to storage if + self._elements[index] = element; } - if (rowLength == 1) return; + // flip bit of depth to get sibling, continue until 2^depth exceeds size + uint256 mask = 2 << depth; - unchecked { - if (colIndex & 1 == 1) { - // sibling is on the left - assembly { - let sibling := sload( - add(keccak256(0x00, 0x20), sub(colIndex, 1)) - ) - mstore(0x00, sibling) - mstore(0x20, hash) - hash := keccak256(0x00, 0x40) - } - } else if (colIndex < rowLength - 1) { - // sibling is on the right (and sibling exists) - assembly { - let sibling := sload( - add(keccak256(0x00, 0x20), add(colIndex, 1)) - ) - mstore(0x00, hash) - mstore(0x20, sibling) - hash := keccak256(0x00, 0x40) - } + if (mask < len) { + uint256 indexLeft = index & ~mask; + uint256 indexRight = index | mask; + + bytes32 nextElement; + + if (index == indexRight) { + nextElement = keccak256( + abi.encodePacked(self._elements[indexLeft], element) + ); + } else if (indexRight < len) { + nextElement = keccak256( + abi.encodePacked(element, self._elements[indexRight]) + ); + } else { + nextElement = element; } - } - _set(self, rowIndex + 1, colIndex >> 1, (rowLength + 1) >> 1, hash); + uint256 nextIndex = indexRight ^ (3 << depth); + + _set(self, depth + 1, nextIndex, nextElement, len); + } } } diff --git a/test/data/IncrementalMerkleTree.ts b/test/data/IncrementalMerkleTree.ts index 2d736ffa0..4be31f291 100644 --- a/test/data/IncrementalMerkleTree.ts +++ b/test/data/IncrementalMerkleTree.ts @@ -35,26 +35,27 @@ describe('IncrementalMerkleTree', () => { }); describe('#height', () => { - it('returns one-indexed height of tree', async () => { - expect(await instance.$height.staticCall(STORAGE_SLOT)).to.equal(0); - - for (let i = 1; i < 10; i++) { + it('returns zero-indexed height of tree', async () => { + for (let i = 0; i < 10; i++) { await instance.$push(STORAGE_SLOT, randomHash()); + const size = await instance.$size.staticCall(STORAGE_SLOT); + expect(await instance.$height.staticCall(STORAGE_SLOT)).to.equal( - Math.ceil(Math.log2(i) + 1), + Math.ceil(Math.log2(Number(size))), ); } }); - }); - describe('#root', () => { - it('returns zero bytes for tree of size zero', async () => { - expect(await instance.$root.staticCall(STORAGE_SLOT)).to.equal( - ethers.ZeroHash, - ); + describe('reverts if', () => { + it('tree size is zero', async () => { + // TODO: reason + await expect(instance.$height.staticCall(STORAGE_SLOT)).to.be.reverted; + }); }); + }); + describe('#root', () => { it('returns contained element for tree of size one', async () => { const hash = randomHash(); @@ -128,6 +129,13 @@ describe('IncrementalMerkleTree', () => { ); } }); + + describe('reverts if', () => { + it('tree size is zero', async () => { + // TODO: reason + await expect(instance.$root.staticCall(STORAGE_SLOT)).to.be.reverted; + }); + }); }); describe('#at', () => { @@ -161,7 +169,7 @@ describe('IncrementalMerkleTree', () => { const hashes: string[] = []; for (let i = 0; i < 10; i++) { - hashes.push(randomHash()); + hashes.push(ethers.zeroPadValue(ethers.toBeHex(i + 1), 32)); } for (let i = 0; i < hashes.length; i++) { @@ -181,11 +189,11 @@ describe('IncrementalMerkleTree', () => { const hashes: string[] = []; for (let i = 0; i < 10; i++) { - hashes.push(randomHash()); + hashes.push(ethers.zeroPadValue(ethers.toBeHex(i + 1), 32)); await instance.$push(STORAGE_SLOT, hashes[i]); } - for (let i = 0; i < hashes.length; i++) { + for (let i = 0; i < hashes.length - 1; i++) { await instance.$pop(STORAGE_SLOT); const tree = new MerkleTree( @@ -193,9 +201,7 @@ describe('IncrementalMerkleTree', () => { keccak256, ); - // MerkleTree library returns truncated zero hash, so must use hexEqual matcher - - expect(await instance.$root.staticCall(STORAGE_SLOT)).to.hexEqual( + expect(await instance.$root.staticCall(STORAGE_SLOT)).to.equal( tree.getHexRoot(), ); } @@ -203,9 +209,11 @@ describe('IncrementalMerkleTree', () => { describe('reverts if', () => { it('tree is size zero', async () => { - await expect(instance.$pop(STORAGE_SLOT)).to.be.revertedWithPanic( - PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, - ); + // TODO: ... + // await expect(instance.$pop(STORAGE_SLOT)).to.be.revertedWithPanic( + // PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, + // ); + await expect(instance.$pop(STORAGE_SLOT)).to.be.reverted; }); }); }); @@ -214,10 +222,9 @@ describe('IncrementalMerkleTree', () => { it('updates Merkle root', async () => { const hashes: string[] = []; - for (let i = 0; i < 10; i++) { - const hash = randomHash(); - hashes.push(hash); - await instance.$push(STORAGE_SLOT, hash); + for (let i = 0; i < 4; i++) { + hashes.push(ethers.zeroPadValue(ethers.toBeHex(i + 1), 32)); + await instance.$push(STORAGE_SLOT, hashes[i]); } for (let i = 0; i < hashes.length; i++) { @@ -239,6 +246,12 @@ describe('IncrementalMerkleTree', () => { await expect( instance.$set.staticCall(STORAGE_SLOT, 0, ethers.ZeroHash), ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS); + + await instance.$push(STORAGE_SLOT, ethers.ZeroHash); + + await expect( + instance.$set.staticCall(STORAGE_SLOT, 1, ethers.ZeroHash), + ).to.be.revertedWithPanic(PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS); }); }); }); From 446c1350cbcfa0e9017ff7bbb546b10abf43af0a Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 13:49:48 -0600 Subject: [PATCH 25/59] remove test function --- contracts/data/IncrementalMerkleTree.sol | 6 ------ 1 file changed, 6 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 37c3f9a0d..c4e74fee3 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -11,12 +11,6 @@ library IncrementalMerkleTree { bytes32[] _elements; } - function elements( - Tree storage self - ) internal view returns (bytes32[] memory els) { - return self._elements; - } - function size(Tree storage self) internal view returns (uint256 treeSize) { treeSize = (self._elements.length + 1) >> 1; } From 15881aea21aa777e1dd478cef0bf799d6a13a8b6 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 13:55:09 -0600 Subject: [PATCH 26/59] add NatSpec comments --- contracts/data/IncrementalMerkleTree.sol | 36 ++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index c4e74fee3..2dc3eef85 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -11,10 +11,20 @@ library IncrementalMerkleTree { bytes32[] _elements; } + /** + * @notice query number of elements contained in tree + * @param self Tree struct storage reference + * @return treeSize size of tree + */ function size(Tree storage self) internal view returns (uint256 treeSize) { treeSize = (self._elements.length + 1) >> 1; } + /** + * @notice query height of tree + * @param self Tree struct storage reference + * @return treeHeight height of tree + */ function height( Tree storage self ) internal view returns (uint256 treeHeight) { @@ -27,12 +37,23 @@ library IncrementalMerkleTree { } } + /** + * @notice query Merkle root + * @param self Tree struct storage reference + * @return rootHash root hash + */ function root(Tree storage self) internal view returns (bytes32 rootHash) { unchecked { return self._elements[(1 << self.height()) - 1]; } } + /** + * @notice retrieve element at given index + * @param self Tree struct storage reference + * @param index index to query + * @return element element stored at index + */ function at( Tree storage self, uint256 index @@ -40,6 +61,11 @@ library IncrementalMerkleTree { element = self._elements[index << 1]; } + /** + * @notice add new element to tree + * @param self Tree struct storage reference + * @param element element to add + */ function push(Tree storage self, bytes32 element) internal { uint256 treeSize = self.size() + 1; uint256 len = (treeSize << 1) - 1; @@ -51,6 +77,10 @@ library IncrementalMerkleTree { _set(self, 0, (treeSize - 1) << 1, element, len); } + /** + * @notice remove last element from tree + * @param self Tree struct storage reference + */ function pop(Tree storage self) internal { uint256 treeSize = self.size() - 1; uint256 len = treeSize == 0 ? 0 : (treeSize << 1) - 1; @@ -64,6 +94,12 @@ library IncrementalMerkleTree { _set(self, 0, (treeSize - 1) << 1, self.at(treeSize - 1), len); } + /** + * @notice overwrite element in tree at given index + * @param self Tree struct storage reference + * @param index index to update + * @param element element to add + */ function set(Tree storage self, uint256 index, bytes32 element) internal { _set(self, 0, index << 1, element, self._elements.length); } From d39a0ac2974874301a18106f8f3dbb1584aa5e35 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 13:55:27 -0600 Subject: [PATCH 27/59] mark _set function as private --- contracts/data/IncrementalMerkleTree.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 2dc3eef85..3b6a6a709 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -110,7 +110,7 @@ library IncrementalMerkleTree { uint256 index, bytes32 element, uint256 len - ) internal { + ) private { if (index < len) { // write element to storage if self._elements[index] = element; From f099c688120c29f6ceea3c2a4a493011cbb05d94 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 14:36:16 -0600 Subject: [PATCH 28/59] use assembly for hashing --- contracts/data/IncrementalMerkleTree.sol | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 3b6a6a709..9928d83fd 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -126,20 +126,24 @@ library IncrementalMerkleTree { bytes32 nextElement; if (index == indexRight) { - nextElement = keccak256( - abi.encodePacked(self._elements[indexLeft], element) - ); + assembly { + mstore(0, self.slot) + mstore(0, sload(add(keccak256(0, 32), indexLeft))) + mstore(32, element) + nextElement := keccak256(0, 64) + } } else if (indexRight < len) { - nextElement = keccak256( - abi.encodePacked(element, self._elements[indexRight]) - ); + assembly { + mstore(0, self.slot) + mstore(32, sload(add(keccak256(0, 32), indexRight))) + mstore(0, element) + nextElement := keccak256(0, 64) + } } else { nextElement = element; } - uint256 nextIndex = indexRight ^ (3 << depth); - - _set(self, depth + 1, nextIndex, nextElement, len); + _set(self, depth + 1, indexRight ^ (3 << depth), nextElement, len); } } } From c990147fefa0037418aa8a9a559f47f4c7df35d3 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 14:37:35 -0600 Subject: [PATCH 29/59] overwrite tree element in stack and pass along --- contracts/data/IncrementalMerkleTree.sol | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 9928d83fd..2ba1623cb 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -123,27 +123,23 @@ library IncrementalMerkleTree { uint256 indexLeft = index & ~mask; uint256 indexRight = index | mask; - bytes32 nextElement; - if (index == indexRight) { assembly { mstore(0, self.slot) mstore(0, sload(add(keccak256(0, 32), indexLeft))) mstore(32, element) - nextElement := keccak256(0, 64) + element := keccak256(0, 64) } } else if (indexRight < len) { assembly { mstore(0, self.slot) mstore(32, sload(add(keccak256(0, 32), indexRight))) mstore(0, element) - nextElement := keccak256(0, 64) + element := keccak256(0, 64) } - } else { - nextElement = element; } - _set(self, depth + 1, indexRight ^ (3 << depth), nextElement, len); + _set(self, depth + 1, indexRight ^ (3 << depth), element, len); } } } From fa93c904dfdd71ccc05c58cee65ee94040e02f07 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 14:55:37 -0600 Subject: [PATCH 30/59] optimize height function --- contracts/data/IncrementalMerkleTree.sol | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 2ba1623cb..1ad0ecac6 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -28,12 +28,14 @@ library IncrementalMerkleTree { function height( Tree storage self ) internal view returns (uint256 treeHeight) { - uint256 treeSize = self.size(); + uint256 length = self._elements.length; - if (treeSize == 0) revert(); + if (length == 0) revert(); - while (1 << treeHeight < treeSize) { - treeHeight++; + while (2 << treeHeight < length) { + unchecked { + treeHeight++; + } } } From 71c8478f7f4d0bcfccbad3a159df55609745f915 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 14:58:38 -0600 Subject: [PATCH 31/59] use assembly to write elements to tree --- contracts/data/IncrementalMerkleTree.sol | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 1ad0ecac6..052ff515d 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -114,8 +114,10 @@ library IncrementalMerkleTree { uint256 len ) private { if (index < len) { - // write element to storage if - self._elements[index] = element; + assembly { + mstore(0, self.slot) + sstore(add(keccak256(0, 32), index), element) + } } // flip bit of depth to get sibling, continue until 2^depth exceeds size From 29e3e3987531b4dec3a607a0f5874e247149b80b Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 15:09:18 -0600 Subject: [PATCH 32/59] calculate array slot in top-level functions and pass down --- contracts/data/IncrementalMerkleTree.sol | 32 ++++++++++++++++-------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 052ff515d..1b2bb0e05 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -76,7 +76,7 @@ library IncrementalMerkleTree { sstore(self.slot, len) } - _set(self, 0, (treeSize - 1) << 1, element, len); + _set(_arraySlot(self), 0, (treeSize - 1) << 1, element, len); } /** @@ -93,7 +93,13 @@ library IncrementalMerkleTree { if (treeSize == 0) return; - _set(self, 0, (treeSize - 1) << 1, self.at(treeSize - 1), len); + _set( + _arraySlot(self), + 0, + (treeSize - 1) << 1, + self.at(treeSize - 1), + len + ); } /** @@ -103,11 +109,18 @@ library IncrementalMerkleTree { * @param element element to add */ function set(Tree storage self, uint256 index, bytes32 element) internal { - _set(self, 0, index << 1, element, self._elements.length); + _set(_arraySlot(self), 0, index << 1, element, self._elements.length); + } + + function _arraySlot(Tree storage self) private pure returns (bytes32 slot) { + assembly { + mstore(0, self.slot) + slot := keccak256(0, 32) + } } function _set( - Tree storage self, + bytes32 arraySlot, uint256 depth, uint256 index, bytes32 element, @@ -115,8 +128,7 @@ library IncrementalMerkleTree { ) private { if (index < len) { assembly { - mstore(0, self.slot) - sstore(add(keccak256(0, 32), index), element) + sstore(add(arraySlot, index), element) } } @@ -129,21 +141,19 @@ library IncrementalMerkleTree { if (index == indexRight) { assembly { - mstore(0, self.slot) - mstore(0, sload(add(keccak256(0, 32), indexLeft))) + mstore(0, sload(add(arraySlot, indexLeft))) mstore(32, element) element := keccak256(0, 64) } } else if (indexRight < len) { assembly { - mstore(0, self.slot) - mstore(32, sload(add(keccak256(0, 32), indexRight))) + mstore(32, sload(add(arraySlot, indexRight))) mstore(0, element) element := keccak256(0, 64) } } - _set(self, depth + 1, indexRight ^ (3 << depth), element, len); + _set(arraySlot, depth + 1, indexRight ^ (3 << depth), element, len); } } } From 01e1c4c4ab4ff95e54ea605fbb3b4b68a5c52f38 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 15:23:45 -0600 Subject: [PATCH 33/59] reorder mstore calls --- contracts/data/IncrementalMerkleTree.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 1b2bb0e05..cbc5cf54e 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -147,8 +147,8 @@ library IncrementalMerkleTree { } } else if (indexRight < len) { assembly { - mstore(32, sload(add(arraySlot, indexRight))) mstore(0, element) + mstore(32, sload(add(arraySlot, indexRight))) element := keccak256(0, 64) } } From f2500fc64bee8cf80544171eb72bbe2887954f4b Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 16:01:39 -0600 Subject: [PATCH 34/59] improve left sibling index calculation --- contracts/data/IncrementalMerkleTree.sol | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index cbc5cf54e..333f0bba4 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -136,8 +136,8 @@ library IncrementalMerkleTree { uint256 mask = 2 << depth; if (mask < len) { - uint256 indexLeft = index & ~mask; uint256 indexRight = index | mask; + uint256 indexLeft = indexRight ^ mask; if (index == indexRight) { assembly { From 24e26632c0028e30b3b8c2d3ccdd42793429c98b Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 16:03:25 -0600 Subject: [PATCH 35/59] calculate left sibling index in assembly --- contracts/data/IncrementalMerkleTree.sol | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 333f0bba4..37af80e58 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -137,11 +137,10 @@ library IncrementalMerkleTree { if (mask < len) { uint256 indexRight = index | mask; - uint256 indexLeft = indexRight ^ mask; if (index == indexRight) { assembly { - mstore(0, sload(add(arraySlot, indexLeft))) + mstore(0, sload(add(arraySlot, xor(indexRight, mask)))) mstore(32, element) element := keccak256(0, 64) } From c45d1b27c89c03989dd3cf4d608c33fe6542c380 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 17:34:04 -0600 Subject: [PATCH 36/59] add comments --- contracts/data/IncrementalMerkleTree.sol | 27 +++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 37af80e58..0d8d65919 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -6,8 +6,6 @@ library IncrementalMerkleTree { using IncrementalMerkleTree for Tree; struct Tree { - // array always has odd length - // elements are stored at even indexes bytes32[] _elements; } @@ -17,6 +15,8 @@ library IncrementalMerkleTree { * @return treeSize size of tree */ function size(Tree storage self) internal view returns (uint256 treeSize) { + // underlying array always has odd length + // elements are stored at even indexes, and their hashes in between treeSize = (self._elements.length + 1) >> 1; } @@ -91,8 +91,11 @@ library IncrementalMerkleTree { sstore(self.slot, len) } + // TODO: do nothing if tree is balanced if (treeSize == 0) return; + // TODO: don't start at depth 0 + _set( _arraySlot(self), 0, @@ -112,6 +115,11 @@ library IncrementalMerkleTree { _set(_arraySlot(self), 0, index << 1, element, self._elements.length); } + /** + * @notice calculate the storage slot of the underlying bytes32 array + * @param self Tree struct storage reference + * @return slot storage slot + */ function _arraySlot(Tree storage self) private pure returns (bytes32 slot) { assembly { mstore(0, self.slot) @@ -127,24 +135,33 @@ library IncrementalMerkleTree { uint256 len ) private { if (index < len) { + // current index is within bounds of data, so write it to storage assembly { sstore(add(arraySlot, index), element) } } - // flip bit of depth to get sibling, continue until 2^depth exceeds size + // lowest n bits will always be (1) for elements at depth n + // flip bit n+1 of an element's index to get it sibling uint256 mask = 2 << depth; if (mask < len) { uint256 indexRight = index | mask; + // if current element is on the left and right element does not exist + // pass element along to next depth unhashed + if (index == indexRight) { + // current element is on the right + // left element is guaranteed to exist assembly { mstore(0, sload(add(arraySlot, xor(indexRight, mask)))) mstore(32, element) element := keccak256(0, 64) } } else if (indexRight < len) { + // current element is on the left + // right element exists assembly { mstore(0, element) mstore(32, sload(add(arraySlot, indexRight))) @@ -152,6 +169,10 @@ library IncrementalMerkleTree { } } + // calculate the index of next element at depth n+1 + // midpoint between current left and right index + // index = indexRight ^ (3 << depth) + _set(arraySlot, depth + 1, indexRight ^ (3 << depth), element, len); } } From 63f684dac3da2826b494aa1e517797f5d39cf257 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 17:35:07 -0600 Subject: [PATCH 37/59] rename len to length --- contracts/data/IncrementalMerkleTree.sol | 28 ++++++++++++++---------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 0d8d65919..a0fb9baf9 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -70,13 +70,13 @@ library IncrementalMerkleTree { */ function push(Tree storage self, bytes32 element) internal { uint256 treeSize = self.size() + 1; - uint256 len = (treeSize << 1) - 1; + uint256 length = (treeSize << 1) - 1; assembly { - sstore(self.slot, len) + sstore(self.slot, length) } - _set(_arraySlot(self), 0, (treeSize - 1) << 1, element, len); + _set(_arraySlot(self), 0, (treeSize - 1) << 1, element, length); } /** @@ -85,10 +85,10 @@ library IncrementalMerkleTree { */ function pop(Tree storage self) internal { uint256 treeSize = self.size() - 1; - uint256 len = treeSize == 0 ? 0 : (treeSize << 1) - 1; + uint256 length = treeSize == 0 ? 0 : (treeSize << 1) - 1; assembly { - sstore(self.slot, len) + sstore(self.slot, length) } // TODO: do nothing if tree is balanced @@ -101,7 +101,7 @@ library IncrementalMerkleTree { 0, (treeSize - 1) << 1, self.at(treeSize - 1), - len + length ); } @@ -132,9 +132,9 @@ library IncrementalMerkleTree { uint256 depth, uint256 index, bytes32 element, - uint256 len + uint256 length ) private { - if (index < len) { + if (index < length) { // current index is within bounds of data, so write it to storage assembly { sstore(add(arraySlot, index), element) @@ -145,7 +145,7 @@ library IncrementalMerkleTree { // flip bit n+1 of an element's index to get it sibling uint256 mask = 2 << depth; - if (mask < len) { + if (mask < length) { uint256 indexRight = index | mask; // if current element is on the left and right element does not exist @@ -159,7 +159,7 @@ library IncrementalMerkleTree { mstore(32, element) element := keccak256(0, 64) } - } else if (indexRight < len) { + } else if (indexRight < length) { // current element is on the left // right element exists assembly { @@ -173,7 +173,13 @@ library IncrementalMerkleTree { // midpoint between current left and right index // index = indexRight ^ (3 << depth) - _set(arraySlot, depth + 1, indexRight ^ (3 << depth), element, len); + _set( + arraySlot, + depth + 1, + indexRight ^ (3 << depth), + element, + length + ); } } } From 1fcd44f0c4bad54aceb7eb5b16743e99fb34b0ac Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 18:40:05 -0600 Subject: [PATCH 38/59] add internal element lookup function --- contracts/data/IncrementalMerkleTree.sol | 32 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index a0fb9baf9..29293b2b6 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -46,7 +46,7 @@ library IncrementalMerkleTree { */ function root(Tree storage self) internal view returns (bytes32 rootHash) { unchecked { - return self._elements[(1 << self.height()) - 1]; + rootHash = _at(_arraySlot(self), (1 << self.height()) - 1); } } @@ -60,7 +60,14 @@ library IncrementalMerkleTree { Tree storage self, uint256 index ) internal view returns (bytes32 element) { - element = self._elements[index << 1]; + if (index >= self.size()) { + assembly { + mstore(0x00, 0x4e487b71) + mstore(0x20, 0x32) + revert(0x1c, 0x24) + } + } + element = _at(_arraySlot(self), index << 1); } /** @@ -96,11 +103,13 @@ library IncrementalMerkleTree { // TODO: don't start at depth 0 + bytes32 slot = _arraySlot(self); + _set( - _arraySlot(self), + slot, 0, (treeSize - 1) << 1, - self.at(treeSize - 1), + _at(slot, (treeSize - 1) << 1), length ); } @@ -127,6 +136,21 @@ library IncrementalMerkleTree { } } + /** + * @notice retreive element at given internal index + * @param arraySlot cached slot of underlying array + * @param index index to query + * @return element element stored at index + */ + function _at( + bytes32 arraySlot, + uint256 index + ) private view returns (bytes32 element) { + assembly { + element := sload(add(arraySlot, index)) + } + } + function _set( bytes32 arraySlot, uint256 depth, From a44dec9684718a5f7b3f4106680e34e4aa7cd5e3 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 19:35:00 -0600 Subject: [PATCH 39/59] increase size of underlying data array by 1 --- contracts/data/IncrementalMerkleTree.sol | 43 +++++++++++++----------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 29293b2b6..128c78420 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -6,6 +6,10 @@ library IncrementalMerkleTree { using IncrementalMerkleTree for Tree; struct Tree { + // underlying array always has even length + // elements are stored at even indexes, and their hashes in between + // last index is empty + bytes32[] _elements; } @@ -15,9 +19,8 @@ library IncrementalMerkleTree { * @return treeSize size of tree */ function size(Tree storage self) internal view returns (uint256 treeSize) { - // underlying array always has odd length - // elements are stored at even indexes, and their hashes in between - treeSize = (self._elements.length + 1) >> 1; + // underlying array is exactly twice the size of the set of leaf nodes + treeSize = self._elements.length >> 1; } /** @@ -76,14 +79,14 @@ library IncrementalMerkleTree { * @param element element to add */ function push(Tree storage self, bytes32 element) internal { - uint256 treeSize = self.size() + 1; - uint256 length = (treeSize << 1) - 1; + uint256 length; assembly { + length := add(sload(self.slot), 2) sstore(self.slot, length) } - _set(_arraySlot(self), 0, (treeSize - 1) << 1, element, length); + _set(_arraySlot(self), 0, length - 2, element, length - 2); } /** @@ -91,27 +94,21 @@ library IncrementalMerkleTree { * @param self Tree struct storage reference */ function pop(Tree storage self) internal { - uint256 treeSize = self.size() - 1; - uint256 length = treeSize == 0 ? 0 : (treeSize << 1) - 1; + uint256 length; assembly { + length := sub(sload(self.slot), 2) sstore(self.slot, length) } // TODO: do nothing if tree is balanced - if (treeSize == 0) return; + if (length == 0) return; // TODO: don't start at depth 0 bytes32 slot = _arraySlot(self); - _set( - slot, - 0, - (treeSize - 1) << 1, - _at(slot, (treeSize - 1) << 1), - length - ); + _set(slot, 0, length - 2, _at(slot, length - 2), length - 2); } /** @@ -121,7 +118,13 @@ library IncrementalMerkleTree { * @param element element to add */ function set(Tree storage self, uint256 index, bytes32 element) internal { - _set(_arraySlot(self), 0, index << 1, element, self._elements.length); + _set( + _arraySlot(self), + 0, + index << 1, + element, + self._elements.length - 2 + ); } /** @@ -158,7 +161,7 @@ library IncrementalMerkleTree { bytes32 element, uint256 length ) private { - if (index < length) { + if (index <= length) { // current index is within bounds of data, so write it to storage assembly { sstore(add(arraySlot, index), element) @@ -169,7 +172,7 @@ library IncrementalMerkleTree { // flip bit n+1 of an element's index to get it sibling uint256 mask = 2 << depth; - if (mask < length) { + if (mask <= length) { uint256 indexRight = index | mask; // if current element is on the left and right element does not exist @@ -183,7 +186,7 @@ library IncrementalMerkleTree { mstore(32, element) element := keccak256(0, 64) } - } else if (indexRight < length) { + } else if (indexRight <= length) { // current element is on the left // right element exists assembly { From a71166d603902ea4d9f3d0d00768f91126a3e22f Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 19:53:12 -0600 Subject: [PATCH 40/59] reduce arithmetic with intermediate variables --- contracts/data/IncrementalMerkleTree.sol | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 128c78420..960e12b3b 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -79,14 +79,15 @@ library IncrementalMerkleTree { * @param element element to add */ function push(Tree storage self, bytes32 element) internal { - uint256 length; + // index of element being added + uint256 index; assembly { - length := add(sload(self.slot), 2) - sstore(self.slot, length) + index := sload(self.slot) + sstore(self.slot, add(index, 2)) } - _set(_arraySlot(self), 0, length - 2, element, length - 2); + _set(_arraySlot(self), 0, index, element, index); } /** @@ -94,21 +95,25 @@ library IncrementalMerkleTree { * @param self Tree struct storage reference */ function pop(Tree storage self) internal { - uint256 length; + // index of element being removed + uint256 index; assembly { - length := sub(sload(self.slot), 2) - sstore(self.slot, length) + index := sub(sload(self.slot), 2) + sstore(self.slot, index) } // TODO: do nothing if tree is balanced - if (length == 0) return; + if (index == 0) return; // TODO: don't start at depth 0 bytes32 slot = _arraySlot(self); - _set(slot, 0, length - 2, _at(slot, length - 2), length - 2); + // index of last element after removal, which may need to be reset + index -= 2; + + _set(slot, 0, index, _at(slot, index), index); } /** From 0fc9192bc13d036e35fae02cd46db909010204df Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 19:54:50 -0600 Subject: [PATCH 41/59] add unchecked block to pop function --- contracts/data/IncrementalMerkleTree.sol | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 960e12b3b..3c1aed066 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -111,7 +111,9 @@ library IncrementalMerkleTree { bytes32 slot = _arraySlot(self); // index of last element after removal, which may need to be reset - index -= 2; + unchecked { + index -= 2; + } _set(slot, 0, index, _at(slot, index), index); } From 2a24a15a740566abfc534a0a30705f7ef1b09e75 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 20:11:16 -0600 Subject: [PATCH 42/59] change _set function argument order --- contracts/data/IncrementalMerkleTree.sol | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 3c1aed066..8c00ecdb2 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -87,7 +87,7 @@ library IncrementalMerkleTree { sstore(self.slot, add(index, 2)) } - _set(_arraySlot(self), 0, index, element, index); + _set(_arraySlot(self), 0, index, index, element); } /** @@ -115,7 +115,7 @@ library IncrementalMerkleTree { index -= 2; } - _set(slot, 0, index, _at(slot, index), index); + _set(slot, 0, index, index, _at(slot, index)); } /** @@ -128,9 +128,9 @@ library IncrementalMerkleTree { _set( _arraySlot(self), 0, + self._elements.length - 2, index << 1, - element, - self._elements.length - 2 + element ); } @@ -164,9 +164,9 @@ library IncrementalMerkleTree { function _set( bytes32 arraySlot, uint256 depth, + uint256 length, uint256 index, - bytes32 element, - uint256 length + bytes32 element ) private { if (index <= length) { // current index is within bounds of data, so write it to storage @@ -210,9 +210,9 @@ library IncrementalMerkleTree { _set( arraySlot, depth + 1, + length, indexRight ^ (3 << depth), - element, - length + element ); } } From dad43bd2c293738b5e2925c862ac63bf63b8f3a4 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 20:12:44 -0600 Subject: [PATCH 43/59] add unchecked block to _set function recursive call --- contracts/data/IncrementalMerkleTree.sol | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 8c00ecdb2..0fb0f241b 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -203,17 +203,19 @@ library IncrementalMerkleTree { } } - // calculate the index of next element at depth n+1 - // midpoint between current left and right index - // index = indexRight ^ (3 << depth) - - _set( - arraySlot, - depth + 1, - length, - indexRight ^ (3 << depth), - element - ); + unchecked { + // calculate the index of next element at depth n+1 + // midpoint between current left and right index + // index = indexRight ^ (3 << depth) + + _set( + arraySlot, + depth + 1, + length, + indexRight ^ (3 << depth), + element + ); + } } } } From 6b1446ac122234019ff9847a1226b05a750aa243 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 20:14:03 -0600 Subject: [PATCH 44/59] rename length to maxIndex --- contracts/data/IncrementalMerkleTree.sol | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 0fb0f241b..a37234edf 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -164,11 +164,11 @@ library IncrementalMerkleTree { function _set( bytes32 arraySlot, uint256 depth, - uint256 length, + uint256 maxIndex, uint256 index, bytes32 element ) private { - if (index <= length) { + if (index <= maxIndex) { // current index is within bounds of data, so write it to storage assembly { sstore(add(arraySlot, index), element) @@ -179,7 +179,7 @@ library IncrementalMerkleTree { // flip bit n+1 of an element's index to get it sibling uint256 mask = 2 << depth; - if (mask <= length) { + if (mask <= maxIndex) { uint256 indexRight = index | mask; // if current element is on the left and right element does not exist @@ -193,7 +193,7 @@ library IncrementalMerkleTree { mstore(32, element) element := keccak256(0, 64) } - } else if (indexRight <= length) { + } else if (indexRight <= maxIndex) { // current element is on the left // right element exists assembly { @@ -211,7 +211,7 @@ library IncrementalMerkleTree { _set( arraySlot, depth + 1, - length, + maxIndex, indexRight ^ (3 << depth), element ); From c57526f0c3613a4db52894f6357187b420d3f6a5 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 20:17:30 -0600 Subject: [PATCH 45/59] panic if nonexistent index is set --- contracts/data/IncrementalMerkleTree.sol | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index a37234edf..bc90ae7eb 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -70,6 +70,7 @@ library IncrementalMerkleTree { revert(0x1c, 0x24) } } + element = _at(_arraySlot(self), index << 1); } @@ -125,6 +126,14 @@ library IncrementalMerkleTree { * @param element element to add */ function set(Tree storage self, uint256 index, bytes32 element) internal { + if (index >= self.size()) { + assembly { + mstore(0x00, 0x4e487b71) + mstore(0x20, 0x32) + revert(0x1c, 0x24) + } + } + _set( _arraySlot(self), 0, From 67e7d22648d947b099e857a5289f1cb506954086 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 20:18:33 -0600 Subject: [PATCH 46/59] add unchecked block to set function --- contracts/data/IncrementalMerkleTree.sol | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index bc90ae7eb..114d1dee0 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -134,13 +134,15 @@ library IncrementalMerkleTree { } } - _set( - _arraySlot(self), - 0, - self._elements.length - 2, - index << 1, - element - ); + unchecked { + _set( + _arraySlot(self), + 0, + self._elements.length - 2, + index << 1, + element + ); + } } /** From cfcc1dd6d3746d7b486bc1bdb02ba7c8c81f8fd3 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 20:46:16 -0600 Subject: [PATCH 47/59] exit pop function if tree is balanced --- contracts/data/IncrementalMerkleTree.sol | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 114d1dee0..e9ff9c512 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -104,18 +104,20 @@ library IncrementalMerkleTree { sstore(self.slot, index) } - // TODO: do nothing if tree is balanced - if (index == 0) return; - - // TODO: don't start at depth 0 + unchecked { + // if tree is now empty, do nothing more + if (index == 0) return; - bytes32 slot = _arraySlot(self); + // if tree is balanced, do nothing more + if (index & (index - 1) == 0) return; - // index of last element after removal, which may need to be reset - unchecked { + // index of last element after removal, which may need to be reset index -= 2; } + bytes32 slot = _arraySlot(self); + + // TODO: don't start at depth 0 _set(slot, 0, index, index, _at(slot, index)); } From 05d6c2a1f876d7406bc696d47615e287a1be1c8e Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 21:06:51 -0600 Subject: [PATCH 48/59] panic on pop of empty tree --- contracts/data/IncrementalMerkleTree.sol | 17 +++++++++++++++-- test/data/IncrementalMerkleTree.ts | 8 +++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index e9ff9c512..629040939 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -96,11 +96,24 @@ library IncrementalMerkleTree { * @param self Tree struct storage reference */ function pop(Tree storage self) internal { - // index of element being removed + // index of next available position in array uint256 index; assembly { - index := sub(sload(self.slot), 2) + index := sload(self.slot) + } + + if (index == 0) { + assembly { + mstore(0x00, 0x4e487b71) + mstore(0x20, 0x32) + revert(0x1c, 0x24) + } + } + + assembly { + // index of element being removed + index := sub(index, 2) sstore(self.slot, index) } diff --git a/test/data/IncrementalMerkleTree.ts b/test/data/IncrementalMerkleTree.ts index 4be31f291..1dcd37c1e 100644 --- a/test/data/IncrementalMerkleTree.ts +++ b/test/data/IncrementalMerkleTree.ts @@ -209,11 +209,9 @@ describe('IncrementalMerkleTree', () => { describe('reverts if', () => { it('tree is size zero', async () => { - // TODO: ... - // await expect(instance.$pop(STORAGE_SLOT)).to.be.revertedWithPanic( - // PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, - // ); - await expect(instance.$pop(STORAGE_SLOT)).to.be.reverted; + await expect(instance.$pop(STORAGE_SLOT)).to.be.revertedWithPanic( + PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, + ); }); }); }); From 539a83b265c586c26546820dc2b54c0d32ac9ca2 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 21:28:33 -0600 Subject: [PATCH 49/59] combine return cases --- contracts/data/IncrementalMerkleTree.sol | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 629040939..a80944ea1 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -118,11 +118,8 @@ library IncrementalMerkleTree { } unchecked { - // if tree is now empty, do nothing more - if (index == 0) return; - - // if tree is balanced, do nothing more - if (index & (index - 1) == 0) return; + // if tree is now empty or is balanced, do nothing more + if (index == 0 || (index & (index - 1) == 0)) return; // index of last element after removal, which may need to be reset index -= 2; From fd804e11f2f5d00f7ce06613f1e16e8aefd47fc7 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Fri, 28 Mar 2025 23:21:13 -0600 Subject: [PATCH 50/59] use Panic library for errors, use POP_ON_EMPTY_ARRAY error --- contracts/data/IncrementalMerkleTree.sol | 20 +++++--------------- test/data/IncrementalMerkleTree.ts | 2 +- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index a80944ea1..17f91a1ee 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -2,6 +2,8 @@ pragma solidity ^0.8.24; +import { Panic } from '../utils/Panic.sol'; + library IncrementalMerkleTree { using IncrementalMerkleTree for Tree; @@ -64,11 +66,7 @@ library IncrementalMerkleTree { uint256 index ) internal view returns (bytes32 element) { if (index >= self.size()) { - assembly { - mstore(0x00, 0x4e487b71) - mstore(0x20, 0x32) - revert(0x1c, 0x24) - } + Panic.panic(Panic.ARRAY_ACCESS_OUT_OF_BOUNDS); } element = _at(_arraySlot(self), index << 1); @@ -104,11 +102,7 @@ library IncrementalMerkleTree { } if (index == 0) { - assembly { - mstore(0x00, 0x4e487b71) - mstore(0x20, 0x32) - revert(0x1c, 0x24) - } + Panic.panic(Panic.POP_ON_EMPTY_ARRAY); } assembly { @@ -139,11 +133,7 @@ library IncrementalMerkleTree { */ function set(Tree storage self, uint256 index, bytes32 element) internal { if (index >= self.size()) { - assembly { - mstore(0x00, 0x4e487b71) - mstore(0x20, 0x32) - revert(0x1c, 0x24) - } + Panic.panic(Panic.ARRAY_ACCESS_OUT_OF_BOUNDS); } unchecked { diff --git a/test/data/IncrementalMerkleTree.ts b/test/data/IncrementalMerkleTree.ts index 1dcd37c1e..1bf2a69de 100644 --- a/test/data/IncrementalMerkleTree.ts +++ b/test/data/IncrementalMerkleTree.ts @@ -210,7 +210,7 @@ describe('IncrementalMerkleTree', () => { describe('reverts if', () => { it('tree is size zero', async () => { await expect(instance.$pop(STORAGE_SLOT)).to.be.revertedWithPanic( - PANIC_CODES.ARRAY_ACCESS_OUT_OF_BOUNDS, + PANIC_CODES.POP_ON_EMPTY_ARRAY, ); }); }); From 7cf3ba441e6a1edcdb241fdd532abb634048ad29 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 13:06:04 -0600 Subject: [PATCH 51/59] fix comment about depth --- contracts/data/IncrementalMerkleTree.sol | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/IncrementalMerkleTree.sol index 17f91a1ee..27ccdb8ce 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/IncrementalMerkleTree.sol @@ -188,8 +188,12 @@ library IncrementalMerkleTree { } } - // lowest n bits will always be (1) for elements at depth n - // flip bit n+1 of an element's index to get it sibling + // all elements at depth n will share the lowest n bits + // these bits also match value of the depth itself + + // create mask of bit n+1 for depth n + // flipping this bit of element's index yields the index of its sibling + // the mask is equal to 2 ** (n + 1) and is also used to determine end of loop uint256 mask = 2 << depth; if (mask <= maxIndex) { From 6b1b2cccedc1b1bc584154193b89428e91b33535 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 20:26:48 -0600 Subject: [PATCH 52/59] rename IncrementalMerkleTree to MerkleTree --- .../{IncrementalMerkleTree.sol => MerkleTree.sol} | 4 ++-- test/data/{IncrementalMerkleTree.ts => MerkleTree.ts} | 11 ++++------- test/inheritance.ts | 2 +- 3 files changed, 7 insertions(+), 10 deletions(-) rename contracts/data/{IncrementalMerkleTree.sol => MerkleTree.sol} (98%) rename test/data/{IncrementalMerkleTree.ts => MerkleTree.ts} (96%) diff --git a/contracts/data/IncrementalMerkleTree.sol b/contracts/data/MerkleTree.sol similarity index 98% rename from contracts/data/IncrementalMerkleTree.sol rename to contracts/data/MerkleTree.sol index 27ccdb8ce..9836c4f88 100644 --- a/contracts/data/IncrementalMerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -4,8 +4,8 @@ pragma solidity ^0.8.24; import { Panic } from '../utils/Panic.sol'; -library IncrementalMerkleTree { - using IncrementalMerkleTree for Tree; +library MerkleTree { + using MerkleTree for Tree; struct Tree { // underlying array always has even length diff --git a/test/data/IncrementalMerkleTree.ts b/test/data/MerkleTree.ts similarity index 96% rename from test/data/IncrementalMerkleTree.ts rename to test/data/MerkleTree.ts index 1bf2a69de..513735cc6 100644 --- a/test/data/IncrementalMerkleTree.ts +++ b/test/data/MerkleTree.ts @@ -1,8 +1,5 @@ import { PANIC_CODES } from '@nomicfoundation/hardhat-chai-matchers/panic'; -import { - $IncrementalMerkleTree, - $IncrementalMerkleTree__factory, -} from '@solidstate/typechain-types'; +import { $MerkleTree, $MerkleTree__factory } from '@solidstate/typechain-types'; import { expect } from 'chai'; import { ethers } from 'hardhat'; import keccak256 from 'keccak256'; @@ -14,12 +11,12 @@ const STORAGE_SLOT = 0n; const randomHash = () => ethers.hexlify(ethers.randomBytes(32)); -describe('IncrementalMerkleTree', () => { - let instance: $IncrementalMerkleTree; +describe('MerkleTree', () => { + let instance: $MerkleTree; beforeEach(async () => { const [deployer] = await ethers.getSigners(); - instance = await new $IncrementalMerkleTree__factory(deployer).deploy(); + instance = await new $MerkleTree__factory(deployer).deploy(); }); describe('#size', () => { diff --git a/test/inheritance.ts b/test/inheritance.ts index 40c4b22c5..bdca5a3a4 100644 --- a/test/inheritance.ts +++ b/test/inheritance.ts @@ -327,7 +327,7 @@ describe('Inheritance Graph', () => { 'PackedDoublyLinkedList', 'EnumerableMap', 'EnumerableSet', - 'IncrementalMerkleTree', + 'MerkleTree', 'CloneFactory', 'Factory', 'MinimalProxyFactory', From 266454f9a479361b8fe252fc94d954f3e47f4d74 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 20:29:25 -0600 Subject: [PATCH 53/59] increase MerkleTree test loop lengths to higher power of 2 --- test/data/MerkleTree.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/data/MerkleTree.ts b/test/data/MerkleTree.ts index 513735cc6..7d6902bbd 100644 --- a/test/data/MerkleTree.ts +++ b/test/data/MerkleTree.ts @@ -23,7 +23,7 @@ describe('MerkleTree', () => { it('returns number of elements in tree', async () => { expect(await instance.$size.staticCall(STORAGE_SLOT)).to.equal(0); - for (let i = 1; i < 10; i++) { + for (let i = 1; i < 2 ** 5; i++) { await instance.$push(STORAGE_SLOT, randomHash()); expect(await instance.$size.staticCall(STORAGE_SLOT)).to.equal(i); @@ -33,7 +33,7 @@ describe('MerkleTree', () => { describe('#height', () => { it('returns zero-indexed height of tree', async () => { - for (let i = 0; i < 10; i++) { + for (let i = 0; i < 2 ** 5; i++) { await instance.$push(STORAGE_SLOT, randomHash()); const size = await instance.$size.staticCall(STORAGE_SLOT); @@ -99,7 +99,7 @@ describe('MerkleTree', () => { }); it('returns result matching reference implementation regardless of previous operations', async () => { - const count = 5; + const count = 2 ** 5; const hashes: string[] = []; for (let i = 0; i < count; i++) { @@ -165,7 +165,7 @@ describe('MerkleTree', () => { it('updates Merkle root', async () => { const hashes: string[] = []; - for (let i = 0; i < 10; i++) { + for (let i = 0; i < 2 ** 5; i++) { hashes.push(ethers.zeroPadValue(ethers.toBeHex(i + 1), 32)); } @@ -185,7 +185,7 @@ describe('MerkleTree', () => { it('updates Merkle root', async () => { const hashes: string[] = []; - for (let i = 0; i < 10; i++) { + for (let i = 0; i < 2 ** 5; i++) { hashes.push(ethers.zeroPadValue(ethers.toBeHex(i + 1), 32)); await instance.$push(STORAGE_SLOT, hashes[i]); } From 7752b6d657d7eabf6893bc2b49f49c17538af14e Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 20:37:32 -0600 Subject: [PATCH 54/59] use bitwise-not instead of subtraction for root calculation --- contracts/data/MerkleTree.sol | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contracts/data/MerkleTree.sol b/contracts/data/MerkleTree.sol index 9836c4f88..61b16fe21 100644 --- a/contracts/data/MerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -51,7 +51,10 @@ library MerkleTree { */ function root(Tree storage self) internal view returns (bytes32 rootHash) { unchecked { - rootHash = _at(_arraySlot(self), (1 << self.height()) - 1); + rootHash = _at( + _arraySlot(self), + ~(type(uint256).max << self.height()) + ); } } From aab3d6263d77fe755a86ba58283a5be36d7a1302 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 21:53:00 -0600 Subject: [PATCH 55/59] move unnecessary assembly operations to solidity --- contracts/data/MerkleTree.sol | 37 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/contracts/data/MerkleTree.sol b/contracts/data/MerkleTree.sol index 61b16fe21..23adf4c05 100644 --- a/contracts/data/MerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -81,14 +81,15 @@ library MerkleTree { * @param element element to add */ function push(Tree storage self, bytes32 element) internal { - // index of element being added - uint256 index; + // index of element being pushed, equal to length of array before operation + uint256 index = self._elements.length; assembly { - index := sload(self.slot) + // increase array length by 2 sstore(self.slot, add(index, 2)) } + // insert element into array and recalculate branch and root nodes _set(_arraySlot(self), 0, index, index, element); } @@ -98,33 +99,32 @@ library MerkleTree { */ function pop(Tree storage self) internal { // index of next available position in array - uint256 index; - - assembly { - index := sload(self.slot) - } + uint256 index = self._elements.length; if (index == 0) { Panic.panic(Panic.POP_ON_EMPTY_ARRAY); } - assembly { - // index of element being removed - index := sub(index, 2) - sstore(self.slot, index) - } - unchecked { - // if tree is now empty or is balanced, do nothing more + // index of element being popped, equal to length of array after operation + index -= 2; + + assembly { + // decrease array length by 2 + sstore(self.slot, index) + } + + // if tree is empty or is balanced after operation, do nothing more if (index == 0 || (index & (index - 1) == 0)) return; - // index of last element after removal, which may need to be reset + // index of last element after operation index -= 2; } bytes32 slot = _arraySlot(self); - // TODO: don't start at depth 0 + // recalculate branch and root nodes + // TODO: this involves an unnecessary storage write at depth 0 _set(slot, 0, index, index, _at(slot, index)); } @@ -173,6 +173,7 @@ library MerkleTree { uint256 index ) private view returns (bytes32 element) { assembly { + // load via assembly to avoid array length check element := sload(add(arraySlot, index)) } } @@ -185,8 +186,8 @@ library MerkleTree { bytes32 element ) private { if (index <= maxIndex) { - // current index is within bounds of data, so write it to storage assembly { + // current index is within bounds of data, so write it to storage sstore(add(arraySlot, index), element) } } From a138d2e98c937e616133822e9e850440302fbef8 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 22:00:13 -0600 Subject: [PATCH 56/59] remove internal calls to size function --- contracts/data/MerkleTree.sol | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/contracts/data/MerkleTree.sol b/contracts/data/MerkleTree.sol index 23adf4c05..dad4a9eb1 100644 --- a/contracts/data/MerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -61,18 +61,21 @@ library MerkleTree { /** * @notice retrieve element at given index * @param self Tree struct storage reference - * @param index index to query + * @param index leaf node index to query * @return element element stored at index */ function at( Tree storage self, uint256 index ) internal view returns (bytes32 element) { - if (index >= self.size()) { + // convert leaf node index to internal index + index <<= 1; + + if (index >= self._elements.length) { Panic.panic(Panic.ARRAY_ACCESS_OUT_OF_BOUNDS); } - element = _at(_arraySlot(self), index << 1); + element = _at(_arraySlot(self), index); } /** @@ -131,22 +134,22 @@ library MerkleTree { /** * @notice overwrite element in tree at given index * @param self Tree struct storage reference - * @param index index to update - * @param element element to add + * @param index leaf node index to update + * @param element element to insert */ function set(Tree storage self, uint256 index, bytes32 element) internal { - if (index >= self.size()) { + uint256 length = self._elements.length; + + // convert leaf node index to internal index + index <<= 1; + + if (index >= length) { Panic.panic(Panic.ARRAY_ACCESS_OUT_OF_BOUNDS); } unchecked { - _set( - _arraySlot(self), - 0, - self._elements.length - 2, - index << 1, - element - ); + // recalculate branch and root nodes + _set(_arraySlot(self), 0, length - 2, index, element); } } From 3cc4a08d47382d2f222ad0c3b03b0bb678a117f3 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 22:03:09 -0600 Subject: [PATCH 57/59] fix comment --- contracts/data/MerkleTree.sol | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/contracts/data/MerkleTree.sol b/contracts/data/MerkleTree.sol index dad4a9eb1..867caf3f9 100644 --- a/contracts/data/MerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -10,8 +10,7 @@ library MerkleTree { struct Tree { // underlying array always has even length // elements are stored at even indexes, and their hashes in between - // last index is empty - + // last index is unused - maintaining an even-length array makes some calculations cheaper bytes32[] _elements; } From 00ff228d9dfcf46ed4e96d8f7fd798cd0c8e21e3 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 22:05:25 -0600 Subject: [PATCH 58/59] use assert for invalid tree height call --- contracts/data/MerkleTree.sol | 2 +- test/data/MerkleTree.ts | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/contracts/data/MerkleTree.sol b/contracts/data/MerkleTree.sol index 867caf3f9..69cfba7ad 100644 --- a/contracts/data/MerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -34,7 +34,7 @@ library MerkleTree { ) internal view returns (uint256 treeHeight) { uint256 length = self._elements.length; - if (length == 0) revert(); + assert(length != 0); while (2 << treeHeight < length) { unchecked { diff --git a/test/data/MerkleTree.ts b/test/data/MerkleTree.ts index 7d6902bbd..f67473a51 100644 --- a/test/data/MerkleTree.ts +++ b/test/data/MerkleTree.ts @@ -46,8 +46,9 @@ describe('MerkleTree', () => { describe('reverts if', () => { it('tree size is zero', async () => { - // TODO: reason - await expect(instance.$height.staticCall(STORAGE_SLOT)).to.be.reverted; + await expect( + instance.$height.staticCall(STORAGE_SLOT), + ).to.be.revertedWithPanic(PANIC_CODES.ASSERTION_ERROR); }); }); }); @@ -129,8 +130,9 @@ describe('MerkleTree', () => { describe('reverts if', () => { it('tree size is zero', async () => { - // TODO: reason - await expect(instance.$root.staticCall(STORAGE_SLOT)).to.be.reverted; + await expect( + instance.$root.staticCall(STORAGE_SLOT), + ).to.be.revertedWithPanic(PANIC_CODES.ASSERTION_ERROR); }); }); }); From db9536532481c11c9cf9a9f77a1e4b5f5b580f36 Mon Sep 17 00:00:00 2001 From: Nick Barry Date: Sat, 29 Mar 2025 22:11:56 -0600 Subject: [PATCH 59/59] replace assertion with Panic library --- contracts/data/MerkleTree.sol | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contracts/data/MerkleTree.sol b/contracts/data/MerkleTree.sol index 69cfba7ad..645d0988c 100644 --- a/contracts/data/MerkleTree.sol +++ b/contracts/data/MerkleTree.sol @@ -34,7 +34,9 @@ library MerkleTree { ) internal view returns (uint256 treeHeight) { uint256 length = self._elements.length; - assert(length != 0); + if (length == 0) { + Panic.panic(Panic.ASSERTION_ERROR); + } while (2 << treeHeight < length) { unchecked {