Backward compatibility hack for Git inputs that depend on Git filters

Before Nix 2.20, we used git, which applies Git filters (in particular
doing end-of-line conversion based on .gitattributes). In 2.20, we
switched to libgit2 and stopped applying filters, which is probably
better for reproducibility. However, that breaks existing lock files /
fetchTree calls for Git inputs that use those filters, since it
invalidates the NAR hash.

So as a backward compatibility hack, we now check the NAR hash
computed over the Git tree without filtering applied. If there is a
hash mismatch, we try again *with* filtering. If that succeeds, we
print a warning and return the filtered tree.

Author: edolstra

src/libfetchers/git.cc

@@ -16,6 +16,7 @@
 #include "nix/util/json-utils.hh"
 #include "nix/util/archive.hh"
 #include "nix/util/mounted-source-accessor.hh"
+#include "nix/fetchers/fetch-to-store.hh"
 
 #include <regex>
 #include <string.h>
@@ -637,6 +638,12 @@ struct GitInputScheme : InputScheme
         return shallow || input.getRevCount().has_value();
     }
 
+    std::string makeFingerprint(const Input & input, const Hash & rev) const
+    {
+        return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")
+               + (getLfsAttr(input) ? ";l" : "");
+    }
+
     std::pair<ref<SourceAccessor>, Input>
     getAccessorFromCommit(const Settings & settings, ref<Store> store, RepoInfo & repoInfo, Input && input) const
     {
@@ -783,6 +790,33 @@ struct GitInputScheme : InputScheme
         bool smudgeLfs = getLfsAttr(input);
         auto accessor = repo->getAccessor(rev, exportIgnore, "«" + input.to_string(true) + "»", smudgeLfs);
 
+        /* Backward compatibility hack for locks produced by Nix < 2.20 that depend on Git filters. Nix >= 2.20 doesn't
+         * apply Git filters, so we may get a NAR hash mismatch. If that happens, try again with filters enabled. */
+        if (auto expectedNarHash = input.getNarHash()) {
+            if (accessor->pathExists(CanonPath(".gitattributes"))) {
+                accessor->fingerprint = makeFingerprint(input, rev);
+                auto narHashNoFilters =
+                    fetchToStore2(settings, *store, {accessor}, FetchMode::DryRun, input.getName()).second;
+                if (expectedNarHash != narHashNoFilters) {
+                    auto accessor2 =
+                        repo->getAccessor(rev, exportIgnore, "«" + input.to_string(true) + "»", smudgeLfs, true);
+                    accessor2->fingerprint = makeFingerprint(input, rev) + ";f";
+                    auto narHashFilters =
+                        fetchToStore2(settings, *store, {accessor2}, FetchMode::DryRun, input.getName()).second;
+                    if (expectedNarHash == narHashFilters) {
+                        warn(
+                            "Git input '%s' specifies a NAR hash '%s' that is only correct if Git filters are applied.\n"
+                            "This is pre-Nix 2.20 behavior; in Nix 2.20 and later, Git filters are not applied.\n"
+                            "Please update the NAR hash to '%s'.",
+                            input.to_string(),
+                            expectedNarHash->to_string(HashFormat::SRI, true),
+                            narHashNoFilters.to_string(HashFormat::SRI, true));
+                        accessor = accessor2;
+                    }
+                }
+            }
+        }
+
         /* If the repo has submodules, fetch them and return a mounted
            input accessor consisting of the accessor for the top-level
            repo and the accessors for the submodules. */
@@ -942,13 +976,8 @@ struct GitInputScheme : InputScheme
 
     std::optional<std::string> getFingerprint(ref<Store> store, const Input & input) const override
     {
-        auto makeFingerprint = [&](const Hash & rev) {
-            return rev.gitRev() + (getSubmodulesAttr(input) ? ";s" : "") + (getExportIgnoreAttr(input) ? ";e" : "")
-                   + (getLfsAttr(input) ? ";l" : "");
-        };
-
         if (auto rev = input.getRev())
-            return makeFingerprint(*rev);
+            return makeFingerprint(input, *rev);
         else {
             auto repoInfo = getRepoInfo(input);
             if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) {
@@ -964,7 +993,7 @@ struct GitInputScheme : InputScheme
                     writeString("deleted:", hashSink);
                     writeString(file.abs(), hashSink);
                 }
-                return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev))
+                return makeFingerprint(input, repoInfo.workdirInfo.headRev.value_or(nullRev))
                        + ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false);
             }
             return std::nullopt;

tests/functional/fetchGit.sh

@@ -310,3 +310,31 @@ git -C "$empty" config user.name "Foobar"
 git -C "$empty" commit --allow-empty --allow-empty-message --message ""
 
 nix eval --impure --expr "let attrs = builtins.fetchGit $empty; in assert attrs.lastModified != 0; assert attrs.rev != \"0000000000000000000000000000000000000000\"; assert attrs.revCount == 1; true"
+
+# Test backward compatibility hack for Nix < 2.20 locks / fetchTree calls that expect Git filters to be applied.
+eol="$TEST_ROOT/git-eol"
+mkdir -p "$eol"
+git init "$eol"
+git -C "$eol" config user.email "[email protected]"
+git -C "$eol" config user.name "Foobar"
+printf "Hello\nWorld\n" > "$eol/crlf"
+git -C "$eol" add crlf
+git -C "$eol" commit -a -m Initial
+printf "crlf text eol=crlf\n" > "$eol/.gitattributes"
+git -C "$eol" add .gitattributes
+git -C "$eol" commit -a -m 'Apply gitattributes'
+
+rev="$(git -C "$eol" rev-parse HEAD)"
+
+export _NIX_TEST_BARF_ON_UNCACHEABLE=1
+
+expectStderr 0 nix eval --expr \
+    "assert builtins.readFile \"\${builtins.fetchTree { type = \"git\"; url = \"file://$eol\"; rev = \"$rev\"; narHash = \"sha256-kxXTCbD8wCYT9qNDKzl9bNKO8++pKY2QwAhqSqENP5k=\"; }}/crlf\" == \"Hello\r\nWorld\r\n\"; true" \
+    | grepQuiet "Please update the NAR hash to 'sha256-LDLvcwdcwCxnuPTxSQ6gLAyopB20lD0bOQoQB3i2hsA='"
+
+nix eval --expr \
+    "assert builtins.readFile \"\${builtins.fetchTree { type = \"git\"; url = \"file://$eol\"; rev = \"$rev\"; narHash = \"sha256-LDLvcwdcwCxnuPTxSQ6gLAyopB20lD0bOQoQB3i2hsA=\"; }}/crlf\" == \"Hello\nWorld\n\"; true"
+
+expectStderr 102 nix eval --expr \
+    "assert builtins.readFile \"\${builtins.fetchTree { type = \"git\"; url = \"file://$eol\"; rev = \"$rev\"; narHash = \"sha256-DLDvcwdcwCxnuPTxSQ6gLAyopB20lD0bOQoQB3i2hsA=\"; }}/crlf\" == \"Hello\nWorld\n\"; true" \
+    | grepQuiet "NAR hash mismatch"