Allow URLs in the Nix search path

E.g. to install "hello" from the latest Nixpkgs:

  $ nix-build '<nixpkgs>' -A hello -I nixpkgs=https://nixos.org/channels/nixpkgs-unstable/nixexprs.tar.xz

Or to install a specific version of NixOS:

  $ nixos-rebuild switch -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/63def04891a0abc328b1b0b3a78ec02c58f48583.tar.gz

src/libexpr/download.cc

@@ -1,6 +1,8 @@
 #include "download.hh"
 #include "util.hh"
 #include "globals.hh"
+#include "hash.hh"
+#include "store-api.hh"
 
 #include <curl/curl.h>
 
@@ -134,4 +136,91 @@ DownloadResult downloadFile(string url, string expectedETag)
     return res;
 }
 
+
+Path downloadFileCached(const string & url, bool unpack)
+{
+    Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
+    createDirs(cacheDir);
+
+    string urlHash = printHash32(hashString(htSHA256, url));
+
+    Path dataFile = cacheDir + "/" + urlHash + ".info";
+    Path fileLink = cacheDir + "/" + urlHash + "-file";
+
+    Path storePath;
+
+    string expectedETag;
+
+    int ttl = settings.get("tarball-ttl", 60 * 60);
+    bool skip = false;
+
+    if (pathExists(fileLink) && pathExists(dataFile)) {
+        storePath = readLink(fileLink);
+        store->addTempRoot(storePath);
+        if (store->isValidPath(storePath)) {
+            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
+            if (ss.size() >= 3 && ss[0] == url) {
+                time_t lastChecked;
+                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
+                    skip = true;
+                else if (!ss[1].empty()) {
+                    printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
+                    expectedETag = ss[1];
+                }
+            }
+        } else
+            storePath = "";
+    }
+
+    string name;
+    auto p = url.rfind('/');
+    if (p != string::npos) name = string(url, p + 1);
+
+    if (!skip) {
+
+        if (storePath.empty())
+            printMsg(lvlInfo, format("downloading ‘%1%’...") % url);
+        else
+            printMsg(lvlInfo, format("checking ‘%1%’...") % url);
+
+        try {
+            auto res = downloadFile(url, expectedETag);
+
+            if (!res.cached)
+                storePath = store->addTextToStore(name, res.data, PathSet(), false);
+
+            assert(!storePath.empty());
+            replaceSymlink(storePath, fileLink);
+
+            writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n");
+        } catch (DownloadError & e) {
+            if (storePath.empty()) throw;
+            printMsg(lvlError, format("warning: %1%; using cached result") % e.msg());
+        }
+    }
+
+    if (unpack) {
+        Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
+        Path unpackedStorePath;
+        if (pathExists(unpackedLink)) {
+            unpackedStorePath = readLink(unpackedLink);
+            store->addTempRoot(unpackedStorePath);
+            if (!store->isValidPath(unpackedStorePath))
+                unpackedStorePath = "";
+        }
+        if (unpackedStorePath.empty()) {
+            printMsg(lvlInfo, format("unpacking ‘%1%’...") % url);
+            Path tmpDir = createTempDir();
+            AutoDelete autoDelete(tmpDir, true);
+            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, "");
+            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, false);
+        }
+        replaceSymlink(unpackedStorePath, unpackedLink);
+        return unpackedStorePath;
+    }
+
+    return storePath;
+}
+
+
 }

src/libexpr/download.hh

@@ -13,6 +13,8 @@ struct DownloadResult
 
 DownloadResult downloadFile(string url, string expectedETag = "");
 
+Path downloadFileCached(const string & url, bool unpack);
+
 MakeError(DownloadError, Error)
 
 }

src/libexpr/parser.y

@@ -527,6 +527,8 @@ formal
 #include <unistd.h>
 
 #include <eval.hh>
+#include <download.hh>
+#include <store-api.hh>
 
 
 namespace nix {
@@ -599,6 +601,15 @@ Expr * EvalState::parseExprFromString(const string & s, const Path & basePath)
 }
 
 
+bool isUri(const string & s)
+{
+    size_t pos = s.find("://");
+    if (pos == string::npos) return false;
+    string scheme(s, 0, pos);
+    return scheme == "http" || scheme == "https";
+}
+
+
 void EvalState::addToSearchPath(const string & s, bool warn)
 {
     size_t pos = s.find('=');
@@ -611,6 +622,9 @@ void EvalState::addToSearchPath(const string & s, bool warn)
         path = string(s, pos + 1);
     }
 
+    if (isUri(path))
+        path = downloadFileCached(path, true);
+
     path = absPath(path);
     if (pathExists(path)) {
         debug(format("adding path ‘%1%’ to the search path") % path);
@@ -629,16 +643,17 @@ Path EvalState::findFile(const string & path)
 
 Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos)
 {
-    foreach (SearchPath::iterator, i, searchPath) {
+    for (auto & i : searchPath) {
+        assert(!isUri(i.second));
         Path res;
-        if (i->first.empty())
+        if (i.first.empty())
-            res = i->second + "/" + path;
+            res = i.second + "/" + path;
         else {
-            if (path.compare(0, i->first.size(), i->first) != 0 ||
+            if (path.compare(0, i.first.size(), i.first) != 0 ||
-                (path.size() > i->first.size() && path[i->first.size()] != '/'))
+                (path.size() > i.first.size() && path[i.first.size()] != '/'))
                 continue;
-            res = i->second +
+            res = i.second +
-                (path.size() == i->first.size() ? "" : "/" + string(path, i->first.size()));
+                (path.size() == i.first.size() ? "" : "/" + string(path, i.first.size()));
         }
         if (pathExists(res)) return canonPath(res);
     }

src/libexpr/primops.cc

@@ -103,7 +103,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
         }
         w.attrs->sort();
         Value fun;
-        state.evalFile(state.findFile("nix/imported-drv-to-derivation.nix"), fun);
+        state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun);
         state.forceFunction(fun, pos);
         mkApp(v, fun, w);
         state.forceAttrs(v, pos);
@@ -1512,88 +1512,7 @@ void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
     } else
         url = state.forceStringNoCtx(*args[0], pos);
 
-    Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
+    mkString(v, downloadFileCached(url, unpack), PathSet({url}));
-    createDirs(cacheDir);
-
-    string urlHash = printHash32(hashString(htSHA256, url));
-
-    Path dataFile = cacheDir + "/" + urlHash + ".info";
-    Path fileLink = cacheDir + "/" + urlHash + "-file";
-
-    Path storePath;
-
-    string expectedETag;
-
-    int ttl = settings.get("tarball-ttl", 60 * 60);
-    bool skip = false;
-
-    if (pathExists(fileLink) && pathExists(dataFile)) {
-        storePath = readLink(fileLink);
-        store->addTempRoot(storePath);
-        if (store->isValidPath(storePath)) {
-            auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
-            if (ss.size() >= 3 && ss[0] == url) {
-                time_t lastChecked;
-                if (string2Int(ss[2], lastChecked) && lastChecked + ttl >= time(0))
-                    skip = true;
-                else if (!ss[1].empty()) {
-                    printMsg(lvlDebug, format("verifying previous ETag ‘%1%’") % ss[1]);
-                    expectedETag = ss[1];
-                }
-            }
-        } else
-            storePath = "";
-    }
-
-    string name;
-    auto p = url.rfind('/');
-    if (p != string::npos) name = string(url, p + 1);
-
-    if (!skip) {
-
-        if (storePath.empty())
-            printMsg(lvlInfo, format("downloading ‘%1%’...") % url);
-        else
-            printMsg(lvlInfo, format("checking ‘%1%’...") % url);
-
-        try {
-            auto res = downloadFile(url, expectedETag);
-
-            if (!res.cached)
-                storePath = store->addTextToStore(name, res.data, PathSet(), state.repair);
-
-            assert(!storePath.empty());
-            replaceSymlink(storePath, fileLink);
-
-            writeFile(dataFile, url + "\n" + res.etag + "\n" + int2String(time(0)) + "\n");
-        } catch (DownloadError & e) {
-            if (storePath.empty()) throw;
-            printMsg(lvlError, format("warning: %1%; using cached result") % e.msg());
-        }
-    }
-
-    if (unpack) {
-        Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked";
-        Path unpackedStorePath;
-        if (pathExists(unpackedLink)) {
-            unpackedStorePath = readLink(unpackedLink);
-            store->addTempRoot(unpackedStorePath);
-            if (!store->isValidPath(unpackedStorePath))
-                unpackedStorePath = "";
-        }
-        if (unpackedStorePath.empty()) {
-            printMsg(lvlDebug, format("unpacking ‘%1%’...") % storePath);
-            Path tmpDir = createTempDir();
-            AutoDelete autoDelete(tmpDir, true);
-            runProgram("tar", true, {"xf", storePath, "-C", tmpDir, "--strip-components", "1"}, "");
-            unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, state.repair);
-        }
-        replaceSymlink(unpackedStorePath, unpackedLink);
-        mkString(v, unpackedStorePath, singleton<PathSet>(unpackedStorePath));
-    }
-
-    else
-        mkString(v, storePath, singleton<PathSet>(storePath));
 }
 
 
@@ -1753,8 +1672,7 @@ void EvalState::createBaseEnv()
 
     /* Add a wrapper around the derivation primop that computes the
        `drvPath' and `outPath' attributes lazily. */
-    string path = findFile("nix/derivation.nix");
+    string path = settings.nixDataDir + "/nix/corepkgs/derivation.nix";
-    assert(!path.empty());
     sDerivationNix = symbols.create(path);
     evalFile(path, v);
     addConstant("derivation", v);

src/nix-env/nix-env.cc

@@ -1423,6 +1423,8 @@ int main(int argc, char * * argv)
 
         if (!op) throw UsageError("no operation specified");
 
+        store = openStore();
+
         globals.state = std::shared_ptr<EvalState>(new EvalState(searchPath));
         globals.state->repair = repair;
 
@@ -1441,8 +1443,6 @@ int main(int argc, char * * argv)
                 : canonPath(settings.nixStateDir + "/profiles/default");
         }
 
-        store = openStore();
-
         op(globals, opFlags, opArgs);
 
         globals.state->printStats();

src/nix-instantiate/nix-instantiate.cc

@@ -155,14 +155,16 @@ int main(int argc, char * * argv)
             return true;
         });
 
+        if (evalOnly && !wantsReadWrite)
+            settings.readOnlyMode = true;
+
+        store = openStore();
+
         EvalState state(searchPath);
         state.repair = repair;
 
         Bindings & autoArgs(*evalAutoArgs(state, autoArgs_));
 
-        if (evalOnly && !wantsReadWrite)
-            settings.readOnlyMode = true;
-
         if (attrPaths.empty()) attrPaths.push_back("");
 
         if (findFile) {
@@ -174,8 +176,6 @@ int main(int argc, char * * argv)
             return;
         }
 
-        store = openStore();
-
         if (readStdin) {
             Expr * e = parseStdin(state);
             processExpr(state, attrPaths, parseOnly, strict, autoArgs,