op-chain-ops: artifacts FS, improve artifacts metadata (#11445)

* op-chain-ops: artifacts FS, improve artifacts metadata

* ci: include artifacts as Go e2e test pre-requisite

* op-chain-ops: move full artifacts test, update testdata

* ci: fix artifacts workspace copy

Author: protolambda

.circleci/config.yml

@@ -857,7 +857,7 @@ jobs:
       - attach_workspace:
           at: /tmp/workspace
       - run:
-          name: Load devnet-allocs
+          name: Load devnet-allocs and artifacts
           command: |
             mkdir -p .devnet
             cp /tmp/workspace/.devnet<<parameters.variant>>/allocs-l2-delta.json .devnet/allocs-l2-delta.json
@@ -866,6 +866,7 @@ jobs:
             cp /tmp/workspace/.devnet<<parameters.variant>>/allocs-l2-granite.json .devnet/allocs-l2-granite.json
             cp /tmp/workspace/.devnet<<parameters.variant>>/allocs-l1.json .devnet/allocs-l1.json
             cp /tmp/workspace/.devnet<<parameters.variant>>/addresses.json .devnet/addresses.json
+            cp -r /tmp/workspace/packages/contracts-bedrock/forge-artifacts packages/contracts-bedrock/forge-artifacts
             cp /tmp/workspace/packages/contracts-bedrock/deploy-config/devnetL1.json packages/contracts-bedrock/deploy-config/devnetL1.json
             cp -r /tmp/workspace/packages/contracts-bedrock/deployments/devnetL1 packages/contracts-bedrock/deployments/devnetL1
       - run:

cannon/mipsevm/testutil/evm.go

@@ -1,6 +1,7 @@
 package testutil
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
 	"math/big"
@@ -12,7 +13,6 @@ import (
 	"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/consensus/ethash"
 	"github.com/ethereum/go-ethereum/core"
@@ -78,7 +78,7 @@ func NewEVMEnv(artifacts *Artifacts, addrs *Addresses) (*vm.EVM, *state.StateDB)
 
 	var mipsCtorArgs [32]byte
 	copy(mipsCtorArgs[12:], addrs.Oracle[:])
-	mipsDeploy := append(hexutil.MustDecode(artifacts.MIPS.Bytecode.Object.String()), mipsCtorArgs[:]...)
+	mipsDeploy := append(bytes.Clone(artifacts.MIPS.Bytecode.Object), mipsCtorArgs[:]...)
 	startingGas := uint64(30_000_000)
 	_, deployedMipsAddr, leftOverGas, err := env.Create(vm.AccountRef(addrs.Sender), mipsDeploy, startingGas, common.U2560)
 	if err != nil {

op-chain-ops/foundry/artifact.go

@@ -1,10 +1,11 @@
 package foundry
 
 import (
+	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/holiman/uint256"
@@ -26,6 +27,7 @@ type Artifact struct {
 	StorageLayout    solc.StorageLayout
 	DeployedBytecode DeployedBytecode
 	Bytecode         Bytecode
+	Metadata         Metadata
 }
 
 func (a *Artifact) UnmarshalJSON(data []byte) error {
@@ -42,6 +44,7 @@ func (a *Artifact) UnmarshalJSON(data []byte) error {
 	a.StorageLayout = artifact.StorageLayout
 	a.DeployedBytecode = artifact.DeployedBytecode
 	a.Bytecode = artifact.Bytecode
+	a.Metadata = artifact.Metadata
 	return nil
 }
 
@@ -51,6 +54,7 @@ func (a Artifact) MarshalJSON() ([]byte, error) {
 		StorageLayout:    a.StorageLayout,
 		DeployedBytecode: a.DeployedBytecode,
 		Bytecode:         a.Bytecode,
+		Metadata:         a.Metadata,
 	}
 	return json.Marshal(artifact)
 }
@@ -62,22 +66,83 @@ type artifactMarshaling struct {
 	StorageLayout    solc.StorageLayout `json:"storageLayout"`
 	DeployedBytecode DeployedBytecode   `json:"deployedBytecode"`
 	Bytecode         Bytecode           `json:"bytecode"`
+	Metadata         Metadata           `json:"metadata"`
+}
+
+// Metadata is the subset of metadata in a foundry contract artifact that we use in OP-Stack tooling.
+type Metadata struct {
+	Compiler struct {
+		Version string `json:"version"`
+	} `json:"compiler"`
+
+	Language string `json:"language"`
+
+	Output json.RawMessage `json:"output"`
+
+	Settings struct {
+		// Remappings of the contract imports
+		Remappings json.RawMessage `json:"remappings"`
+		// Optimizer settings affect the compiler output, but can be arbitrary.
+		// We load them opaquely, to include it in the hash of what we run.
+		Optimizer json.RawMessage `json:"optimizer"`
+		// Metadata is loaded opaquely, similar to the Optimizer, to include in hashing.
+		// E.g. the bytecode-hash contract suffix as setting is enabled/disabled in here.
+		Metadata json.RawMessage `json:"metadata"`
+		// Map of full contract path to compiled contract name.
+		CompilationTarget map[string]string `json:"compilationTarget"`
+		// EVM version affects output, and hence included.
+		EVMVersion string `json:"evmVersion"`
+		// Libraries data
+		Libraries json.RawMessage `json:"libraries"`
+	} `json:"settings"`
+
+	Sources map[string]ContractSource `json:"sources"`
+
+	Version int `json:"version"`
+}
+
+// ContractSource represents a JSON value in the "sources" map of a contract metadata dump.
+// This uniquely identifies the source code of the contract.
+type ContractSource struct {
+	Keccak256 common.Hash `json:"keccak256"`
+	URLs      []string    `json:"urls"`
+	License   string      `json:"license"`
+}
+
+var ErrLinkingUnsupported = errors.New("cannot load bytecode with linking placeholders")
+
+// LinkableBytecode is not purely hex, it returns an ErrLinkingUnsupported error when
+// input contains __$aaaaaaa$__ style linking placeholders.
+// See https://docs.soliditylang.org/en/latest/using-the-compiler.html#library-linking
+// In practice this is only used by test contracts to link in large test libraries.
+type LinkableBytecode []byte
+
+func (lb *LinkableBytecode) UnmarshalJSON(data []byte) error {
+	if bytes.Contains(data, []byte("__$")) {
+		return ErrLinkingUnsupported
+	}
+	return (*hexutil.Bytes)(lb).UnmarshalJSON(data)
+}
+
+func (lb LinkableBytecode) MarshalText() ([]byte, error) {
+	return (hexutil.Bytes)(lb).MarshalText()
 }
 
 // DeployedBytecode represents the deployed bytecode section of the solc compiler output.
 type DeployedBytecode struct {
-	SourceMap           string          `json:"sourceMap"`
-	Object              hexutil.Bytes   `json:"object"`
-	LinkReferences      json.RawMessage `json:"linkReferences"`
-	ImmutableReferences json.RawMessage `json:"immutableReferences,omitempty"`
+	SourceMap           string           `json:"sourceMap"`
+	Object              LinkableBytecode `json:"object"`
+	LinkReferences      json.RawMessage  `json:"linkReferences"`
+	ImmutableReferences json.RawMessage  `json:"immutableReferences,omitempty"`
 }
 
 // Bytecode represents the bytecode section of the solc compiler output.
 type Bytecode struct {
-	SourceMap           string          `json:"sourceMap"`
-	Object              hexutil.Bytes   `json:"object"`
-	LinkReferences      json.RawMessage `json:"linkReferences"`
-	ImmutableReferences json.RawMessage `json:"immutableReferences,omitempty"`
+	SourceMap string `json:"sourceMap"`
+	// not purely hex, can contain __$aaaaaaa$__ style linking placeholders
+	Object              LinkableBytecode `json:"object"`
+	LinkReferences      json.RawMessage  `json:"linkReferences"`
+	ImmutableReferences json.RawMessage  `json:"immutableReferences,omitempty"`
 }
 
 // ReadArtifact will read an artifact from disk given a path.
@@ -130,15 +195,14 @@ func (d *ForgeAllocs) UnmarshalJSON(b []byte) error {
 }
 
 func LoadForgeAllocs(allocsPath string) (*ForgeAllocs, error) {
-	path := filepath.Join(allocsPath)
-	f, err := os.OpenFile(path, os.O_RDONLY, 0644)
+	f, err := os.OpenFile(allocsPath, os.O_RDONLY, 0644)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open forge allocs %q: %w", path, err)
+		return nil, fmt.Errorf("failed to open forge allocs %q: %w", allocsPath, err)
 	}
 	defer f.Close()
 	var out ForgeAllocs
 	if err := json.NewDecoder(f).Decode(&out); err != nil {
-		return nil, fmt.Errorf("failed to json-decode forge allocs %q: %w", path, err)
+		return nil, fmt.Errorf("failed to json-decode forge allocs %q: %w", allocsPath, err)
 	}
 	return &out, nil
 }

op-chain-ops/foundry/artifact_test.go

@@ -10,13 +10,13 @@ import (
 
 // TestArtifactJSON tests roundtrip serialization of a foundry artifact for commonly used fields.
 func TestArtifactJSON(t *testing.T) {
-	artifact, err := ReadArtifact("testdata/OptimismPortal.json")
+	artifact, err := ReadArtifact("testdata/forge-artifacts/Owned.sol/Owned.json")
 	require.NoError(t, err)
 
 	data, err := json.Marshal(artifact)
 	require.NoError(t, err)
 
-	file, err := os.ReadFile("testdata/OptimismPortal.json")
+	file, err := os.ReadFile("testdata/forge-artifacts/Owned.sol/Owned.json")
 	require.NoError(t, err)
 
 	got := unmarshalIntoMap(t, data)
@@ -26,6 +26,7 @@ func TestArtifactJSON(t *testing.T) {
 	require.JSONEq(t, marshal(t, got["deployedBytecode"]), marshal(t, expected["deployedBytecode"]))
 	require.JSONEq(t, marshal(t, got["abi"]), marshal(t, expected["abi"]))
 	require.JSONEq(t, marshal(t, got["storageLayout"]), marshal(t, expected["storageLayout"]))
+	require.JSONEq(t, marshal(t, got["metadata"]), marshal(t, expected["metadata"]))
 }
 
 func unmarshalIntoMap(t *testing.T, file []byte) map[string]any {

op-chain-ops/foundry/artifactsfs.go

@@ -0,0 +1,89 @@
+package foundry
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"strings"
+)
+
+type statDirFs interface {
+	fs.StatFS
+	fs.ReadDirFS
+}
+
+func OpenArtifactsDir(dirPath string) *ArtifactsFS {
+	dir := os.DirFS(dirPath)
+	if d, ok := dir.(statDirFs); !ok {
+		panic("Go DirFS guarantees changed")
+	} else {
+		return &ArtifactsFS{FS: d}
+	}
+}
+
+// ArtifactsFS wraps a filesystem (read-only access) of a forge-artifacts bundle.
+// The root contains directories for every artifact,
+// each containing one or more entries (one per solidity compiler version) for a solidity contract.
+// See OpenArtifactsDir for reading from a local directory.
+// Alternative FS systems, like a tarball, may be used too.
+type ArtifactsFS struct {
+	FS statDirFs
+}
+
+func (af *ArtifactsFS) ListArtifacts() ([]string, error) {
+	entries, err := af.FS.ReadDir(".")
+	if err != nil {
+		return nil, fmt.Errorf("failed to list artifacts: %w", err)
+	}
+	out := make([]string, 0, len(entries))
+	for _, d := range entries {
+		if name := d.Name(); strings.HasSuffix(name, ".sol") {
+			out = append(out, strings.TrimSuffix(name, ".sol"))
+		}
+	}
+	return out, nil
+}
+
+// ListContracts lists the contracts of the named artifact.
+// E.g. "Owned" might list "Owned.0.8.15", "Owned.0.8.25", and "Owned".
+func (af *ArtifactsFS) ListContracts(name string) ([]string, error) {
+	f, err := af.FS.Open(name + ".sol")
+	if err != nil {
+		return nil, fmt.Errorf("failed to open artifact %q: %w", name, err)
+	}
+	defer f.Close()
+	dirFile, ok := f.(fs.ReadDirFile)
+	if !ok {
+		return nil, fmt.Errorf("no dir for artifact %q, but got %T", name, f)
+	}
+	entries, err := dirFile.ReadDir(0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to list artifact contents of %q: %w", name, err)
+	}
+	out := make([]string, 0, len(entries))
+	for _, d := range entries {
+		if name := d.Name(); strings.HasSuffix(name, ".json") {
+			out = append(out, strings.TrimSuffix(name, ".json"))
+		}
+	}
+	return out, nil
+}
+
+// ReadArtifact reads a specific JSON contract artifact from the FS.
+// The contract name may be suffixed by a solidity compiler version, e.g. "Owned.0.8.25".
+func (af *ArtifactsFS) ReadArtifact(name string, contract string) (*Artifact, error) {
+	artifactPath := path.Join(name+".sol", contract+".json")
+	f, err := af.FS.Open(artifactPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open artifact %q: %w", artifactPath, err)
+	}
+	defer f.Close()
+	dec := json.NewDecoder(f)
+	var out Artifact
+	if err := dec.Decode(&out); err != nil {
+		return nil, fmt.Errorf("failed to decode artifact %q: %w", name, err)
+	}
+	return &out, nil
+}

op-chain-ops/foundry/artifactsfs_test.go

@@ -0,0 +1,42 @@
+package foundry
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/log"
+
+	"github.com/ethereum-optimism/optimism/op-service/testlog"
+)
+
+func TestArtifacts(t *testing.T) {
+	logger := testlog.Logger(t, log.LevelWarn) // lower this log level to get verbose test dump of all artifacts
+	af := OpenArtifactsDir("./testdata/forge-artifacts")
+	artifacts, err := af.ListArtifacts()
+	require.NoError(t, err)
+	require.NotEmpty(t, artifacts)
+	for _, name := range artifacts {
+		contracts, err := af.ListContracts(name)
+		require.NoError(t, err, "failed to list %s", name)
+		require.NotEmpty(t, contracts)
+		for _, contract := range contracts {
+			artifact, err := af.ReadArtifact(name, contract)
+			if err != nil {
+				if errors.Is(err, ErrLinkingUnsupported) {
+					logger.Info("linking not supported", "name", name, "contract", contract, "err", err)
+					continue
+				}
+				require.NoError(t, err, "failed to read artifact %s / %s", name, contract)
+			}
+			logger.Info("artifact",
+				"name", name,
+				"contract", contract,
+				"compiler", artifact.Metadata.Compiler.Version,
+				"sources", len(artifact.Metadata.Sources),
+				"evmVersion", artifact.Metadata.Settings.EVMVersion,
+			)
+		}
+	}
+}