Verification flow refactoring (#1845)

This is the main pull request for the verification flow refactoring.
Smaller feature-specific pull requests were merged into this branch
before it was finalized.

See #1665 for more details

Author: marcocastignoli

packages/lib-sourcify/README.md

@@ -18,7 +18,7 @@
     "check": "run-s check:*",
     "check:eslint": "eslint . --ext .ts",
     "check:prettier": "prettier \"./**/*.ts\" --check",
-    "test": "c8 --reporter=none mocha -r ts-node/register test/**/*.spec.ts --no-timeout --exit",
+    "test": "c8 --reporter=none mocha -r ts-node/register test/**/*.spec.ts test/*.spec.ts --no-timeout --exit",
     "check-cli": "run-s test diff-integration-tests check-integration-tests",
     "check-integration-tests": "run-s check-integration-test:*",
     "diff-integration-tests": "mkdir -p diff && rm -rf diff/test && cp -r test diff/test && rm -rf diff/test/test-*/.git && cd diff && git init --quiet && git add -A && git commit --quiet --no-verify --allow-empty -m 'WIP' && echo '\\n\\nCommitted most recent integration test output in the \"diff\" directory. Review the changes with \"cd diff && git diff HEAD\" or your preferred git diff viewer.'",

packages/lib-sourcify/package.json

@@ -0,0 +1,156 @@
+import { AuxdataStyle } from '@ethereum-sourcify/bytecode-utils';
+import {
+  CompilationTarget,
+  CompiledContractCborAuxdata,
+  Metadata,
+  CompilationLanguage,
+  StringMap,
+  LinkReferences,
+  CompilationError,
+} from './CompilationTypes';
+import {
+  ImmutableReferences,
+  ISolidityCompiler,
+  SolidityJsonInput,
+  SolidityOutput,
+  SolidityOutputContract,
+} from './SolidityTypes';
+import {
+  IVyperCompiler,
+  VyperJsonInput,
+  VyperOutput,
+  VyperOutputContract,
+} from './VyperTypes';
+import { logInfo, logSilly, logWarn } from '../logger';
+
+export abstract class AbstractCompilation {
+  /**
+   * Constructor parameters
+   */
+  abstract compiler: ISolidityCompiler | IVyperCompiler;
+  abstract compilerVersion: string;
+  abstract compilationTarget: CompilationTarget;
+  abstract jsonInput: SolidityJsonInput | VyperJsonInput;
+
+  protected _metadata?: Metadata;
+  compilerOutput?: SolidityOutput | VyperOutput;
+
+  abstract auxdataStyle: AuxdataStyle;
+  abstract language: CompilationLanguage;
+
+  /** Marks the positions of the CborAuxdata parts in the bytecode */
+  protected _creationBytecodeCborAuxdata?: CompiledContractCborAuxdata;
+  protected _runtimeBytecodeCborAuxdata?: CompiledContractCborAuxdata;
+
+  /**
+   * Recompiles the contract with the specified compiler settings
+   * @param forceEmscripten Whether to force using the WebAssembly binary for compilation (only for Solidity)
+   */
+  abstract compile(forceEmscripten?: boolean): Promise<void>;
+  abstract generateCborAuxdataPositions(
+    forceEmscripten?: boolean,
+  ): Promise<void>;
+
+  public async compileAndReturnCompilationTarget(
+    forceEmscripten = false,
+  ): Promise<SolidityOutputContract | VyperOutputContract> {
+    const version = this.compilerVersion;
+
+    const compilationStartTime = Date.now();
+    logInfo('Compiling contract', {
+      version,
+      contract: this.compilationTarget.name,
+      path: this.compilationTarget.path,
+      forceEmscripten,
+    });
+    logSilly('Compilation input', { solcJsonInput: this.jsonInput });
+    this.compilerOutput = await this.compiler.compile(
+      version,
+      this.jsonInput as any,
+      forceEmscripten,
+    );
+    if (this.compilerOutput === undefined) {
+      logWarn('Compiler error', {
+        errorMessages: ['compilerOutput is undefined'],
+      });
+      throw new CompilationError('Compiler error', 'compiler_error');
+    }
+
+    // We call contractCompilerOutput() before logging because it can throw an error
+    const compilationTargetContract = this.contractCompilerOutput;
+
+    const compilationEndTime = Date.now();
+    const compilationDuration = compilationEndTime - compilationStartTime;
+    logSilly('Compilation output', { compilerOutput: this.compilerOutput });
+    logInfo('Compiled contract', {
+      version,
+      contract: this.compilationTarget.name,
+      path: this.compilationTarget.path,
+      forceEmscripten,
+      compilationDuration: `${compilationDuration}ms`,
+    });
+
+    return compilationTargetContract;
+  }
+
+  get contractCompilerOutput(): SolidityOutputContract | VyperOutputContract {
+    if (!this.compilerOutput) {
+      logWarn('Compiler output is undefined');
+      throw new Error('Compiler output is undefined');
+    }
+    if (
+      !this.compilerOutput.contracts ||
+      !this.compilerOutput.contracts[this.compilationTarget.path] ||
+      !this.compilerOutput.contracts[this.compilationTarget.path][
+        this.compilationTarget.name
+      ]
+    ) {
+      const error = new Error('Contract not found in compiler output');
+      logWarn('Contract not found in compiler output');
+      throw error;
+    }
+    return this.compilerOutput.contracts[this.compilationTarget.path][
+      this.compilationTarget.name
+    ];
+  }
+
+  get creationBytecode() {
+    return `0x${this.contractCompilerOutput.evm.bytecode.object}`;
+  }
+
+  get runtimeBytecode() {
+    return `0x${this.contractCompilerOutput.evm.deployedBytecode.object}`;
+  }
+
+  get metadata() {
+    if (!this._metadata) {
+      throw new Error('Metadata is not set');
+    }
+    return this._metadata;
+  }
+
+  get sources() {
+    return Object.keys(this.jsonInput.sources).reduce((acc, source) => {
+      acc[source] = this.jsonInput.sources[source].content;
+      return acc;
+    }, {} as StringMap);
+  }
+
+  abstract get immutableReferences(): ImmutableReferences;
+  abstract get runtimeLinkReferences(): LinkReferences;
+  abstract get creationLinkReferences(): LinkReferences;
+
+  get creationBytecodeCborAuxdata(): CompiledContractCborAuxdata {
+    if (!this._creationBytecodeCborAuxdata) {
+      throw new Error('Creation bytecode cbor auxdata is not set');
+    }
+    return this._creationBytecodeCborAuxdata;
+  }
+
+  get runtimeBytecodeCborAuxdata(): CompiledContractCborAuxdata {
+    if (!this._runtimeBytecodeCborAuxdata) {
+      throw new Error('Runtime bytecode cbor auxdata is not set');
+    }
+    return this._runtimeBytecodeCborAuxdata;
+  }
+}

packages/lib-sourcify/src/Compilation/AbstractCompilation.ts

@@ -0,0 +1,140 @@
+import { Abi } from 'abitype';
+import { SoliditySettings } from './SolidityTypes';
+import { SourcifyLibError } from '../SourcifyLibError';
+
+export interface LinkReferences {
+  [filePath: string]: {
+    [libraryName: string]: [
+      {
+        length: number;
+        start: number;
+      },
+    ];
+  };
+}
+
+export interface MetadataSource {
+  keccak256: string;
+  content?: string;
+  urls?: string[];
+  license?: string;
+}
+
+export interface MetadataSourceMap {
+  [index: string]: MetadataSource;
+}
+
+export interface Devdoc {
+  author?: string;
+  details?: string;
+  errors?: {
+    [index: string]: {
+      details?: string;
+    };
+  };
+  events?: {
+    [index: string]: {
+      details?: string;
+      params?: any;
+    };
+  };
+  kind: 'dev';
+  methods: {
+    [index: string]: {
+      details?: string;
+      params?: any;
+      returns?: any;
+    };
+  };
+  stateVariables?: any;
+  title?: string;
+  version?: number;
+}
+
+export interface Userdoc {
+  errors?: {
+    [index: string]: {
+      notice?: string;
+    }[];
+  };
+  events?: {
+    [index: string]: {
+      notice?: string;
+    };
+  };
+  kind: 'user';
+  methods: {
+    [index: string]: {
+      notice: string;
+    };
+  };
+  version?: number;
+}
+
+export interface MetadataOutput {
+  abi: Abi;
+  devdoc?: Devdoc;
+  userdoc?: Userdoc;
+}
+
+// Metadata JSON's "settings" does have extra "compilationTarget" and its "libraries" field is in a different format
+// ( libraries["MyContract.sol:Mycontract"]:"0xab..cd" vs libraries["MyContract.sol"]["MyContract"]:"0xab..cd")
+export interface MetadataCompilerSettings
+  extends Omit<SoliditySettings, 'libraries' | 'outputSelection'> {
+  compilationTarget: {
+    [sourceName: string]: string;
+  };
+  libraries?: {
+    [index: string]: string;
+  };
+}
+
+// Metadata type that reflects the metadata object from
+// https://docs.soliditylang.org/en/latest/metadata.html
+export interface Metadata {
+  compiler: {
+    keccak256?: string;
+    version: string;
+  };
+  language: string;
+  output: MetadataOutput;
+  settings: MetadataCompilerSettings;
+  sources: MetadataSourceMap;
+  version: number;
+}
+
+export interface CompiledContractCborAuxdata {
+  [index: string]: {
+    offset: number;
+    value: string;
+  };
+}
+
+export interface StringMap {
+  [key: string]: string;
+}
+
+export interface AuxdataDiff {
+  real: string;
+  diffStart: number;
+  diff: string;
+}
+
+export interface CompilationTarget {
+  name: string;
+  path: string;
+}
+
+export type CompilationLanguage = 'Solidity' | 'Vyper';
+
+export type CompilationErrorCode =
+  | 'cannot_generate_cbor_auxdata_positions'
+  | 'invalid_compiler_version'
+  | 'contract_not_found_in_compiler_output'
+  | 'compiler_error';
+
+export class CompilationError extends SourcifyLibError {
+  constructor(message: string, code: CompilationErrorCode) {
+    super(message, code);
+  }
+}