From 7d89e46f65fb5cafa6528c3ff39e4a53d83e99ec Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 14:24:57 +0200 Subject: [PATCH 1/2] Sync: Support moving out of another Sync --- src/libutil/include/nix/util/sync.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 0c3e1f52836..4b9d546d2b7 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -39,6 +39,7 @@ public: SyncBase() { } SyncBase(const T & data) : data(data) { } SyncBase(T && data) noexcept : data(std::move(data)) { } + SyncBase(SyncBase && other) noexcept : data(std::move(*other.lock())) { } template class Lock From efcb9e36a9b5d6e31ce3513d5fd0571757c41c9b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 14 May 2025 14:17:57 +0200 Subject: [PATCH 2/2] Remove global fetcher cache The cache is now part of fetchers::Settings. --- src/libcmd/common-eval-args.cc | 20 ++++++++++++++++--- src/libcmd/installable-value.cc | 2 +- src/libexpr/eval.cc | 3 ++- src/libexpr/primops.cc | 1 + src/libexpr/primops/fetchTree.cc | 3 ++- src/libfetchers/cache.cc | 9 ++++++--- src/libfetchers/fetch-to-store.cc | 12 ++++++----- src/libfetchers/fetchers.cc | 2 +- src/libfetchers/git-utils.cc | 7 ++++--- src/libfetchers/git.cc | 16 +++++++-------- src/libfetchers/github.cc | 10 +++++----- src/libfetchers/include/nix/fetchers/cache.hh | 2 -- .../include/nix/fetchers/fetch-settings.hh | 9 +++++++++ .../include/nix/fetchers/fetch-to-store.hh | 1 + .../include/nix/fetchers/git-utils.hh | 4 ++-- .../include/nix/fetchers/tarball.hh | 1 + src/libfetchers/mercurial.cc | 10 +++++----- src/libfetchers/path.cc | 3 ++- src/libfetchers/registry.cc | 2 +- src/libfetchers/tarball.cc | 16 +++++++++------ src/libflake/flake.cc | 4 ++-- src/nix-channel/nix-channel.cc | 8 +++++--- src/nix/flake.cc | 2 +- 23 files changed, 93 insertions(+), 54 deletions(-) diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e087678c26f..d275beb12c3 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -34,7 +34,12 @@ EvalSettings evalSettings { auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); debug("fetching flake search path element '%s''", rest); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); - auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + auto storePath = nix::fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy, + lockedRef.input.getName()); state.allowPath(storePath); return state.storePath(storePath); }, @@ -177,7 +182,11 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas state.store, state.fetchSettings, EvalSettings::resolvePseudoUrl(s)); - auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); + auto storePath = fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy); return state.storePath(storePath); } @@ -185,7 +194,12 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); - auto storePath = nix::fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy, lockedRef.input.getName()); + auto storePath = nix::fetchToStore( + state.fetchSettings, + *state.store, + SourcePath(accessor), + FetchMode::Copy, + lockedRef.input.getName()); state.allowPath(storePath); return state.storePath(storePath); } diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index d9ac3a29e7a..e92496347e0 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -45,7 +45,7 @@ ref InstallableValue::require(ref installable) std::optional InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std::string_view errorCtx) { if (v.type() == nPath) { - auto storePath = fetchToStore(*state->store, v.path(), FetchMode::Copy); + auto storePath = fetchToStore(state->fetchSettings, *state->store, v.path(), FetchMode::Copy); return {{ .path = DerivedPath::Opaque { .path = std::move(storePath), diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ab7f33d5f89..4c3aec977e6 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -2423,6 +2423,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat ? *dstPathCached : [&]() { auto dstPath = fetchToStore( + fetchSettings, *store, path.resolveSymlinks(SymlinkResolution::Ancestors), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, @@ -3125,7 +3126,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat store, fetchSettings, EvalSettings::resolvePseudoUrl(value)); - auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); + auto storePath = fetchToStore(fetchSettings, *store, SourcePath(accessor), FetchMode::Copy); return finish(this->storePath(storePath)); } catch (Error & e) { logWarning({ diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 535a9a50117..d13314ffba9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2545,6 +2545,7 @@ static void addPath( if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { auto dstPath = fetchToStore( + state.fetchSettings, *state.store, path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 745705e04c1..f262507dffa 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -537,11 +537,12 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto storePath = unpack ? fetchToStore( + state.fetchSettings, *state.store, fetchers::downloadTarball(state.store, state.fetchSettings, *url), FetchMode::Copy, name) - : fetchers::downloadFile(state.store, *url, name).storePath; + : fetchers::downloadFile(state.store, state.fetchSettings, *url, name).storePath; if (expectedHash) { auto hash = unpack diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index d369d213f51..9a2531ba526 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "nix/fetchers/cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/users.hh" #include "nix/store/sqlite.hh" #include "nix/util/sync.hh" @@ -162,10 +163,12 @@ struct CacheImpl : Cache } }; -ref getCache() +ref Settings::getCache() const { - static auto cache = std::make_shared(); - return ref(cache); + auto cache(_cache.lock()); + if (!*cache) + *cache = std::make_shared(); + return ref(*cache); } } diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index f1b02f4e0a8..f7ab32322ef 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -1,13 +1,14 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix { fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string &name, - const std::string &fingerprint, + const std::string & name, + const std::string & fingerprint, ContentAddressMethod method, - const std::string &path) + const std::string & path) { return fetchers::Cache::Key{"fetchToStore", { {"name", name}, @@ -19,6 +20,7 @@ fetchers::Cache::Key makeFetchToStoreCacheKey( } StorePath fetchToStore( + const fetchers::Settings & settings, Store & store, const SourcePath & path, FetchMode mode, @@ -34,7 +36,7 @@ StorePath fetchToStore( if (!filter && path.accessor->fingerprint) { cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); - if (auto res = fetchers::getCache()->lookupStorePath(*cacheKey, store)) { + if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) { debug("store path cache hit for '%s'", path); return res->storePath; } @@ -56,7 +58,7 @@ StorePath fetchToStore( debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); if (cacheKey && mode == FetchMode::Copy) - fetchers::getCache()->upsert(*cacheKey, store, {}, storePath); + settings.getCache()->upsert(*cacheKey, store, {}, storePath); return storePath; } diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index e0161dee8fe..35201e8aef1 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -198,7 +198,7 @@ std::pair Input::fetchToStore(ref store) const try { auto [accessor, result] = getAccessorUnchecked(store); - auto storePath = nix::fetchToStore(*store, SourcePath(accessor), FetchMode::Copy, result.getName()); + auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); auto narHash = store->queryPathInfo(storePath)->narHash; result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 58ba398896b..60d8edee06b 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -1,6 +1,7 @@ #include "nix/fetchers/git-utils.hh" #include "nix/fetchers/git-lfs-fetch.hh" #include "nix/fetchers/cache.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/util/finally.hh" #include "nix/util/processes.hh" #include "nix/util/signals.hh" @@ -610,18 +611,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); } - Hash treeHashToNarHash(const Hash & treeHash) override + Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) override { auto accessor = getAccessor(treeHash, false, ""); fetchers::Cache::Key cacheKey{"treeHashToNarHash", {{"treeHash", treeHash.gitRev()}}}; - if (auto res = fetchers::getCache()->lookup(cacheKey)) + if (auto res = settings.getCache()->lookup(cacheKey)) return Hash::parseAny(fetchers::getStrAttr(*res, "narHash"), HashAlgorithm::SHA256); auto narHash = accessor->hashPath(CanonPath::root); - fetchers::getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); + settings.getCache()->upsert(cacheKey, fetchers::Attrs({{"narHash", narHash.to_string(HashFormat::SRI, true)}})); return narHash; } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 492867dc404..f716094b6ae 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -480,11 +480,11 @@ struct GitInputScheme : InputScheme return repoInfo; } - uint64_t getLastModified(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const + uint64_t getLastModified(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitLastModified", {{"rev", rev.gitRev()}}}; - auto cache = getCache(); + auto cache = settings.getCache(); if (auto res = cache->lookup(key)) return getIntAttr(*res, "lastModified"); @@ -496,11 +496,11 @@ struct GitInputScheme : InputScheme return lastModified; } - uint64_t getRevCount(const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const + uint64_t getRevCount(const Settings & settings, const RepoInfo & repoInfo, const std::filesystem::path & repoDir, const Hash & rev) const { Cache::Key key{"gitRevCount", {{"rev", rev.gitRev()}}}; - auto cache = getCache(); + auto cache = settings.getCache(); if (auto revCountAttrs = cache->lookup(key)) return getIntAttr(*revCountAttrs, "revCount"); @@ -678,12 +678,12 @@ struct GitInputScheme : InputScheme Attrs infoAttrs({ {"rev", rev.gitRev()}, - {"lastModified", getLastModified(repoInfo, repoDir, rev)}, + {"lastModified", getLastModified(*input.settings, repoInfo, repoDir, rev)}, }); if (!getShallowAttr(input)) infoAttrs.insert_or_assign("revCount", - getRevCount(repoInfo, repoDir, rev)); + getRevCount(*input.settings, repoInfo, repoDir, rev)); printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.locationToArg()); @@ -799,7 +799,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign("rev", rev.gitRev()); input.attrs.insert_or_assign("revCount", - rev == nullRev ? 0 : getRevCount(repoInfo, repoPath, rev)); + rev == nullRev ? 0 : getRevCount(*input.settings, repoInfo, repoPath, rev)); verifyCommit(input, repo); } else { @@ -818,7 +818,7 @@ struct GitInputScheme : InputScheme input.attrs.insert_or_assign( "lastModified", repoInfo.workdirInfo.headRev - ? getLastModified(repoInfo, repoPath, *repoInfo.workdirInfo.headRev) + ? getLastModified(*input.settings, repoInfo, repoPath, *repoInfo.workdirInfo.headRev) : 0); return {accessor, std::move(input)}; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 4fcad5f2631..511afabe979 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -265,7 +265,7 @@ struct GitArchiveInputScheme : InputScheme input.attrs.erase("ref"); input.attrs.insert_or_assign("rev", rev->gitRev()); - auto cache = getCache(); + auto cache = input.settings->getCache(); Cache::Key treeHashKey{"gitRevToTreeHash", {{"rev", rev->gitRev()}}}; Cache::Key lastModifiedKey{"gitRevToLastModified", {{"rev", rev->gitRev()}}}; @@ -407,7 +407,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", headers).storePath))); + downloadFile(store, *input.settings, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), @@ -481,7 +481,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", headers).storePath))); + downloadFile(store, *input.settings, url, "source", headers).storePath))); if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo { @@ -551,7 +551,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); + downloadFile(store, *input.settings, fmt("%s/HEAD", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; getline(is, line); @@ -567,7 +567,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); + downloadFile(store, *input.settings, fmt("%s/info/refs", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 5b9319d774b..6ac693183f9 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -91,6 +91,4 @@ struct Cache Store & store) = 0; }; -ref getCache(); - } diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 54c42084344..7bd04db53b0 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -3,6 +3,8 @@ #include "nix/util/types.hh" #include "nix/util/configuration.hh" +#include "nix/util/ref.hh" +#include "nix/util/sync.hh" #include #include @@ -11,6 +13,8 @@ namespace nix::fetchers { +struct Cache; + struct Settings : public Config { Settings(); @@ -110,6 +114,11 @@ struct Settings : public Config When empty, disables the global flake registry. )", {}, true, Xp::Flakes}; + + ref getCache() const; + +private: + mutable Sync> _cache; }; } diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index 44c33c147ed..a52d567ecfb 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -15,6 +15,7 @@ enum struct FetchMode { DryRun, Copy }; * Copy the `path` to the Nix store. */ StorePath fetchToStore( + const fetchers::Settings & settings, Store & store, const SourcePath & path, FetchMode mode, diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 1506f8509e4..2926deb4f44 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -5,7 +5,7 @@ namespace nix { -namespace fetchers { struct PublicKey; } +namespace fetchers { struct PublicKey; struct Settings; } /** * A sink that writes into a Git repository. Note that nothing may be written @@ -115,7 +115,7 @@ struct GitRepo * Given a Git tree hash, compute the hash of its NAR * serialisation. This is memoised on-disk. */ - virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + virtual Hash treeHashToNarHash(const fetchers::Settings & settings, const Hash & treeHash) = 0; /** * If the specified Git object is a directory with a single entry diff --git a/src/libfetchers/include/nix/fetchers/tarball.hh b/src/libfetchers/include/nix/fetchers/tarball.hh index 691142091fa..2c5ea209f01 100644 --- a/src/libfetchers/include/nix/fetchers/tarball.hh +++ b/src/libfetchers/include/nix/fetchers/tarball.hh @@ -26,6 +26,7 @@ struct DownloadFileResult DownloadFileResult downloadFile( ref store, + const Settings & settings, const std::string & url, const std::string & name, const Headers & headers = {}); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 74e9fd08998..0b63876deae 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -253,13 +253,13 @@ struct MercurialInputScheme : InputScheme }}; if (!input.getRev()) { - if (auto res = getCache()->lookupWithTTL(refToRevKey)) + if (auto res = input.settings->getCache()->lookupWithTTL(refToRevKey)) input.attrs.insert_or_assign("rev", getRevAttr(*res, "rev").gitRev()); } /* If we have a rev, check if we have a cached store path. */ if (auto rev = input.getRev()) { - if (auto res = getCache()->lookupStorePath(revInfoKey(*rev), *store)) + if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(*rev), *store)) return makeResult(res->value, res->storePath); } @@ -309,7 +309,7 @@ struct MercurialInputScheme : InputScheme /* Now that we have the rev, check the cache again for a cached store path. */ - if (auto res = getCache()->lookupStorePath(revInfoKey(rev), *store)) + if (auto res = input.settings->getCache()->lookupStorePath(revInfoKey(rev), *store)) return makeResult(res->value, res->storePath); Path tmpDir = createTempDir(); @@ -326,9 +326,9 @@ struct MercurialInputScheme : InputScheme }); if (!origRev) - getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); + input.settings->getCache()->upsert(refToRevKey, {{"rev", rev.gitRev()}}); - getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); + input.settings->getCache()->upsert(revInfoKey(rev), *store, infoAttrs, storePath); return makeResult(infoAttrs, std::move(storePath)); } diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 670397cb6b1..9239fd27466 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -4,6 +4,7 @@ #include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/cache.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix::fetchers { @@ -149,7 +150,7 @@ struct PathInputScheme : InputScheme auto fp = getFingerprint(store, input); if (fp) { auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - fetchers::getCache()->upsert(cacheKey, *store, {}, *storePath); + input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath); } /* Trust the lastModified value supplied by the user, if diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index bfaf9569a4e..335935f53af 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -156,7 +156,7 @@ static std::shared_ptr getGlobalRegistry(const Settings & settings, re } if (!isAbsolute(path)) { - auto storePath = downloadFile(store, path, "flake-registry.json").storePath; + auto storePath = downloadFile(store, settings, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 1bd7e3e5947..b0822cc3301 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -9,11 +9,13 @@ #include "nix/fetchers/store-path-accessor.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/git-utils.hh" +#include "nix/fetchers/fetch-settings.hh" namespace nix::fetchers { DownloadFileResult downloadFile( ref store, + const Settings & settings, const std::string & url, const std::string & name, const Headers & headers) @@ -25,7 +27,7 @@ DownloadFileResult downloadFile( {"name", name}, }}}; - auto cached = getCache()->lookupStorePath(key, *store); + auto cached = settings.getCache()->lookupStorePath(key, *store); auto useCached = [&]() -> DownloadFileResult { @@ -92,7 +94,7 @@ DownloadFileResult downloadFile( key.second.insert_or_assign("url", url); assert(!res.urls.empty()); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); - getCache()->upsert(key, *store, infoAttrs, *storePath); + settings.getCache()->upsert(key, *store, infoAttrs, *storePath); } return { @@ -104,13 +106,14 @@ DownloadFileResult downloadFile( } static DownloadTarballResult downloadTarball_( + const Settings & settings, const std::string & url, const Headers & headers, const std::string & displayPrefix) { Cache::Key cacheKey{"tarball", {{"url", url}}}; - auto cached = getCache()->lookupExpired(cacheKey); + auto cached = settings.getCache()->lookupExpired(cacheKey); auto attrsToResult = [&](const Attrs & infoAttrs) { @@ -196,7 +199,7 @@ static DownloadTarballResult downloadTarball_( /* Insert a cache entry for every URL in the redirect chain. */ for (auto & url : res->urls) { cacheKey.second.insert_or_assign("url", url); - getCache()->upsert(cacheKey, infoAttrs); + settings.getCache()->upsert(cacheKey, infoAttrs); } // FIXME: add a cache entry for immutableUrl? That could allow @@ -341,7 +344,7 @@ struct FileInputScheme : CurlInputScheme the Nix store directly, since there is little deduplication benefit in using the Git cache for single big files like tarballs. */ - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); + auto file = downloadFile(store, *input.settings, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store->queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); @@ -373,6 +376,7 @@ struct TarballInputScheme : CurlInputScheme auto input(_input); auto result = downloadTarball_( + *input.settings, getStrAttr(input.attrs, "url"), {}, "«" + input.to_string() + "»"); @@ -390,7 +394,7 @@ struct TarballInputScheme : CurlInputScheme input.attrs.insert_or_assign("lastModified", uint64_t(result.lastModified)); input.attrs.insert_or_assign("narHash", - getTarballCache()->treeHashToNarHash(result.treeHash).to_string(HashFormat::SRI, true)); + getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); return {result.accessor, input}; } diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index 987c9f610af..3277cc05bad 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -30,7 +30,7 @@ static StorePath copyInputToStore( const fetchers::Input & originalInput, ref accessor) { - auto storePath = fetchToStore(*state.store, accessor, FetchMode::Copy, input.getName()); + auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName()); state.allowPath(storePath); @@ -276,7 +276,7 @@ static Flake readFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - auto storePath = fetchToStore(*state.store, setting.value->path(), FetchMode::Copy); + auto storePath = fetchToStore(state.fetchSettings, *state.store, setting.value->path(), FetchMode::Copy); flake.config.settings.emplace( state.symbols[setting.name], state.store->printStorePath(storePath)); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9db13022081..bc1164a4345 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -4,9 +4,11 @@ #include "nix/store/filetransfer.hh" #include "nix/store/store-open.hh" #include "nix/cmd/legacy.hh" +#include "nix/cmd/common-eval-args.hh" #include "nix/expr/eval-settings.hh" // for defexpr #include "nix/util/users.hh" #include "nix/fetchers/tarball.hh" +#include "nix/fetchers/fetch-settings.hh" #include "self-exe.hh" #include "man-pages.hh" @@ -114,7 +116,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); + auto result = fetchers::downloadFile(store, fetchSettings, url, std::string(baseNameOf(url))); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -128,9 +130,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); + filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); + filename = store->toRealPath(fetchers::downloadFile(store, fetchSettings, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression diff --git a/src/nix/flake.cc b/src/nix/flake.cc index ea83bb54c84..fdeedf9c1ff 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -1488,7 +1488,7 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON auto originalRef = getFlakeRef(); auto resolvedRef = originalRef.resolve(store); auto [accessor, lockedRef] = resolvedRef.lazyFetch(store); - auto storePath = fetchToStore(*store, accessor, FetchMode::Copy, lockedRef.input.getName()); + auto storePath = fetchToStore(getEvalState()->fetchSettings, *store, accessor, FetchMode::Copy, lockedRef.input.getName()); auto hash = store->queryPathInfo(storePath)->narHash; if (json) {