Merge pull request #11690 from DeterminateSystems/non-contiguous-tarballs

Handle tarballs where directory entries are not contiguous

Author: edolstra

src/libfetchers/git-utils.cc

@@ -977,8 +977,24 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink
 
     void pushBuilder(std::string name)
     {
+        const git_tree_entry * entry;
+        Tree prevTree = nullptr;
+
+        if (!pendingDirs.empty() &&
+            (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str())))
+        {
+            /* Clone a tree that we've already finished. This happens
+               if a tarball has directory entries that are not
+               contiguous. */
+            if (git_tree_entry_type(entry) != GIT_OBJECT_TREE)
+                throw Error("parent of '%s' is not a directory", name);
+
+            if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry))
+                throw Error("looking up parent of '%s': %s", name, git_error_last()->message);
+        }
+
         git_treebuilder * b;
-        if (git_treebuilder_new(&b, *repo, nullptr))
+        if (git_treebuilder_new(&b, *repo, prevTree.get()))
             throw Error("creating a tree builder: %s", git_error_last()->message);
         pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) });
     };

tests/functional/tarball.sh

@@ -97,3 +97,17 @@ chmod +x "$TEST_ROOT/tar_root/foo"
 tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" .
 path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)"
 [[ $(cat "$path/foo") = bar ]]
+
+# Test a tarball with non-contiguous directory entries.
+rm -rf "$TEST_ROOT/tar_root"
+mkdir -p "$TEST_ROOT/tar_root/a/b"
+echo foo > "$TEST_ROOT/tar_root/a/b/foo"
+echo bla > "$TEST_ROOT/tar_root/bla"
+tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" .
+echo abc > "$TEST_ROOT/tar_root/bla"
+echo xyzzy > "$TEST_ROOT/tar_root/a/b/xyzzy"
+tar rvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" ./a/b/xyzzy ./bla
+path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)"
+[[ $(cat "$path/a/b/xyzzy") = xyzzy ]]
+[[ $(cat "$path/a/b/foo") = foo ]]
+[[ $(cat "$path/bla") = abc ]]