checkpoint commit

Switched from overrideDerivation to overrideAttrs because
overrideDerivation was not working on Russell's new PDF derivation --
see BlockstreamResearch/simplicity#145

Russell explained on Rocket that overrideDerivation is "old school"
and soft-deprecated and that overrideAttrs is to be prefered. See
https://nixos.org/manual/nixpkgs/stable/#sec-pkg-overrideDerivation
(It references "the overrideAttrs documentation". To find this, scroll
up.)

Author: apoelstra

BlockstreamResearch.Simplicity.check-pr.nix

@@ -7,6 +7,8 @@
 , stdenv ? pkgs.stdenv
 , jsonConfigFile
 , prNum
+# Only used by checkHEad, not checkPr
+, singleRev ? prNum
 }:
 let
   utils = import ./andrew-utils.nix { };
@@ -31,7 +33,7 @@ let
         srcName = self: self.src.commitId;
         mtxName = self: "${self.src.shortId}-${self.attr}";
 
-        attr = [ "coq" "haskell" "compcert" "vst" ];
+        attr = [ "coq" "haskell" "compcert" "vst" "pdf" ];
         doCheck = null;
         wideMultiply = null;
         withCoverage = null;
@@ -105,7 +107,7 @@ let
               '';
             })
         else drv;
-        taggedDrv = diffDrv.overrideDerivation (drv: {
+        taggedDrv = diffDrv.overrideAttrs (drv: {
           # Add a bunch of stuff just to make the derivation easier to grok
           checkPrProjectName = "Simplicity";
           checkPrAttr = attr;
@@ -127,7 +129,7 @@ in
           src = builtins.fetchGit {
             allRefs = true;
             url = jsonConfig.gitDir;
-            rev = prNum;
+            rev = singleRev;
           };
           name = builtins.toString prNum;
           shortId = name;

BlockstreamResearch.rust-simplicity.check-pr.nix

@@ -7,6 +7,8 @@
 , stdenv ? pkgs.stdenv
 , jsonConfigFile
 , prNum
+# Only used by checkHEad, not checkPr
+, singleRev ? prNum
 }:
 let
   utils = import ./andrew-utils.nix { };
@@ -16,7 +18,7 @@ let
     (pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default))
     pkgs.rust-bin.stable.latest.default
     pkgs.rust-bin.beta.latest.default
-    pkgs.rust-bin.stable."1.41.0".default
+    pkgs.rust-bin.stable."1.48.0".default
   ];
   isNightly = rustc: rustc == builtins.head allRustcs;
   gitCommits = utils.githubPrSrcs {
@@ -152,7 +154,7 @@ in
             gitSrc = builtins.fetchGit {
               allRefs = true;
               url = jsonConfig.gitDir;
-              rev = prNum;
+              rev = singleRev;
             };
           in rec {
               src = if isSys

BlockstreamResearch.secp256k1-zkp.check-pr.nix

@@ -4,6 +4,8 @@
 , stdenv ? pkgs.stdenv
 , jsonConfigFile
 , prNum
+# Only used by checkHEad, not checkPr
+, singleRev ? prNum
 }:
 let
   utils = import ./andrew-utils.nix { };
@@ -137,7 +139,7 @@ in
           src = builtins.fetchGit {
             allRefs = true;
             url = jsonConfig.gitDir;
-            rev = prNum;
+            rev = singleRev;
           };
           name = builtins.toString prNum;
           shortId = name;

ElementsProject.elements-miniscript.check-pr.nix

@@ -7,6 +7,8 @@
 , stdenv ? pkgs.stdenv
 , jsonConfigFile
 , prNum
+# Only used by checkHEad, not checkPr
+, singleRev ? prNum
 }:
 let
   utils = import ./andrew-utils.nix { };
@@ -15,7 +17,7 @@ let
     (pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default))
     pkgs.rust-bin.stable.latest.default
     pkgs.rust-bin.beta.latest.default
-    pkgs.rust-bin.stable."1.41.0".default
+    pkgs.rust-bin.stable."1.48.0".default
   ];
   isNightly = rustc: rustc == builtins.head allRustcs;
   gitCommits = utils.githubPrSrcs {
@@ -120,7 +122,7 @@ in
           src = builtins.fetchGit {
             allRefs = true;
             url = jsonConfig.gitDir;
-            rev = prNum;
+            rev = singleRev;
           };
           name = builtins.toString prNum;
           shortId = name;

ElementsProject.rust-elements.check-pr.nix

@@ -7,6 +7,8 @@
 , stdenv ? pkgs.stdenv
 , jsonConfigFile
 , prNum
+# Only used by checkHEad, not checkPr
+, singleRev ? prNum
 }:
 let
   utils = import ./andrew-utils.nix { };
@@ -15,7 +17,7 @@ let
     (pkgs.rust-bin.selectLatestNightlyWith (toolchain: toolchain.default))
     pkgs.rust-bin.stable.latest.default
     pkgs.rust-bin.beta.latest.default
-    pkgs.rust-bin.stable."1.41.0".default
+    pkgs.rust-bin.stable."1.48.0".default
   ];
   isNightly = rustc: rustc == builtins.head allRustcs;
   gitCommits = utils.githubPrSrcs {
@@ -107,6 +109,7 @@ let
                 pushd ${nixes.generated}/crate
                 #cargo clippy --locked -- -D warnings
                 #cargo fmt --all -- --check
+                #RUSTDOCFLAGS="--cfg docsrs -D warnings -D rustdoc::broken-intra-doc-links" cargo doc --all-features
                 popd
               ''
               else "";
@@ -134,7 +137,7 @@ in
           src = builtins.fetchGit {
             allRefs = true;
             url = jsonConfig.gitDir;
-            rev = prNum;
+            rev = singleRev;
           };
           name = builtins.toString prNum;
           shortId = name;

andrew-utils.nix

@@ -195,12 +195,15 @@ rec {
   # Cargo.nix as the key, and the result of calling it as the value.
   #
   # Assumes that your matrix has entries projectName, prNum, rustc, lockFile, src.
+  #
+  # Note that EVERY INPUT TO THIS FUNCTION MUST BE ADDED TO generatedCargoNix. If it is
+  # not, the memoization logic will collapse all the different values for that input
+  # into one.
   crate2nixSingleCheckMemo =
     { projectName
     , prNum
     , rustc
     , lockFile
-    , features ? []
     , src
     , ...
     }:
@@ -211,7 +214,7 @@ rec {
         }) ];
       };
       generatedCargoNix = tools-nix.generatedCargoNix {
-        name = "${projectName}-generated-cargo-nix-${builtins.toString prNum}-${src.shortId}";
+        name = "${projectName}-generated-cargo-nix-${builtins.toString prNum}-${src.shortId}-${builtins.toString rustc}";
         src = src.src;
         overrideLockFile = lockFile;
       };
@@ -235,7 +238,6 @@ rec {
               echo "rustc: ${builtins.toString rustc}"
               echo "lockFile: ${lockFile}"
               echo "Source: ${builtins.toJSON src}"
-              echo "Features: ${builtins.toJSON features}"
             '';
           }
           else pkgs.buildRustCrate crate;
@@ -258,6 +260,78 @@ rec {
         called = calledCargoNix;
       };
     };
+
+  cargoFuzzDrv = {
+    normalDrv
+  , projectName
+  , src
+  , nixes
+  , fuzzTargets
+  }: let
+    overlaidPkgs = import <nixpkgs> {
+      overlays = [
+        (import (fetchTarball "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"))
+      ];
+    };
+    singleFuzzDrv = fuzzTarget: stdenv.mkDerivation {
+      name = "fuzz-${fuzzTarget}";
+      src = src.src;
+      buildInputs = [
+        overlaidPkgs.rust-bin.stable."1.58.0".default
+        (import ./honggfuzz-rs.nix { })
+        # Pinned version because of breaking change in args to init_disassemble_info
+        nixpkgs.libopcodes_2_38 # for dis-asm.h and bfd.h
+        nixpkgs.libunwind       # for libunwind-ptrace.h
+      ];
+      phases = [ "unpackPhase" "buildPhase" ];
+
+      buildPhase = ''
+        set -x
+        export CARGO_HOME=$PWD/cargo
+        export HFUZZ_RUN_ARGS="--run_time 300 --exit_upon_crash"
+
+        cargo -V
+        cargo hfuzz version
+        echo "Source: ${builtins.toJSON src}"
+        echo "Fuzz target: ${fuzzTarget}"
+
+        # honggfuzz rebuilds the world, including itself for some reason, and
+        # it expects to be able to build itself in-place. So we need a read/write
+        # copy.
+        cp -r ${nixes.generated}/cargo .
+        chmod -R +w cargo
+
+        DEP_DIR=$(grep 'directory =' $CARGO_HOME/config | sed 's/directory = "\(.*\)"/\1/')
+        cp -r "$DEP_DIR" vendor-copy/
+        chmod +w vendor-copy/
+        rm vendor-copy/*honggfuzz*
+        cp -rL "$DEP_DIR/"*honggfuzz* vendor-copy/ # -L means copy soft-links as real files
+        chmod -R +w vendor-copy/
+        # These two lines are just a search-and-replace ... but trying to get sed to replace
+        # one string full of slashes with another is an unreadable mess, so easier to just
+        # erase the line completely then recreate it with echo.
+        sed -i "s/directory = \".*\"//" "$CARGO_HOME/config"
+        echo "directory = \"$PWD/vendor-copy\"" >> "$CARGO_HOME/config"
+        cat "$CARGO_HOME/config"
+        # Done crazy honggfuzz shit
+
+        pushd fuzz/
+        cargo hfuzz run "${fuzzTarget}"
+        popd
+
+        touch $out
+      '';
+    };
+    fuzzDrv = overlaidPkgs.linkFarm
+      "${projectName}-${src.shortId}-fuzz" 
+      ((map (x: rec {
+        name = "fuzz-${path.name}";
+        path = singleFuzzDrv x;
+      }) fuzzTargets) ++ [{
+        name = "fuzz-normal-tests";
+        path = normalDrv;
+      }]);
+    in fuzzDrv;
 }
 
 

bitcoin.secp256k1.check-pr.nix

@@ -4,6 +4,8 @@
 , stdenv ? pkgs.stdenv
 , jsonConfigFile
 , prNum
+# Only used by checkHEad, not checkPr
+, singleRev ? prNum
 }:
 let
   utils = import ./andrew-utils.nix { };
@@ -30,15 +32,16 @@ let
         ["recovery"]
       ];
       ecmultGenPrecision = [ 2 4 8 ];
-      ecmultWindow = [
+      # z-prefix this to try to spread out the 24-bit instances in the matrix
+      zecmultWindow = [
         2 10 15
         # Frustratingly my system OOMs when dealing with more than one commit
         # at 24-bit ecmult windows. Even with 1TB of RAM. So throttle it in
         # the multi-commit case but leave it be for the "just testing HEAD"
         # case.
-        if builtins.length gitCommits > 1
+        (if builtins.length gitCommits > 1
         then 21
-        else 24
+        else 24)
       ];
       withAsm = [ "no" "x86_64" ];
       withMsan = [ true false ];
@@ -53,7 +56,7 @@ let
       mtxName,
       extraModules,
       ecmultGenPrecision,
-      ecmultWindow,
+      zecmultWindow,
       withAsm,
       withMsan,
       widemul,
@@ -79,17 +82,17 @@ let
           fi
         '';
       # clang can't seem to handle ecmult windows > 20 :(
-      adjEcmultWindow = if withMsan && ecmultWindow > 20
-        then ecmultWindow - 4
-        else ecmultWindow;
+      adjEcmultWindow = if withMsan && zecmultWindow > 20
+        then zecmultWindow - 4
+        else zecmultWindow;
       drv = stdenv.mkDerivation {
         name = "${projectName}-${src.shortId}";
         src = src.src;
 
         nativeBuildInputs = [ pkgs.pkgconfig pkgs.autoreconfHook pkgs.valgrind ]
           ++ lib.optionals withMsan [
-            pkgs.llvmPackages_15.llvm # to get llvm-symbolizer when clang blows up
-            pkgs.clang_15
+            pkgs.llvmPackages_16.llvm # to get llvm-symbolizer when clang blows up
+            pkgs.clang_16
           ];
         buildInputs = [];
 
@@ -145,12 +148,13 @@ in
           src = builtins.fetchGit {
             allRefs = true;
             url = jsonConfig.gitDir;
-            rev = prNum;
+            rev = singleRev;
           };
           name = builtins.toString prNum;
           shortId = name;
           commitId = shortId;
         };
+        zecmultWindow = [2 10 15 23]; # fixme drop this when we sort out OOM issues
       })
       checkData.argsMatrices;
   });