feat(glue): Add hashed_mirrors support to eval fetcher

This change adds basic scaffolding to allow configuring hashed_mirrors that will be used
by fetchurl to download artifacts by their hash, this is useful in case certain URLs are
no longer available but required to bootstrap nixpkgs stdenv.

These urls will have higher priority than the url specified in fetchurl(and friends) and
will be attempted before falling back to the actual url specified in fetchurl.

Change-Id: I589bdef609075f274cbdf6b26af602cafaa7496a
Reviewed-on: https://cl.snix.dev/c/snix/+/30567
Tested-by: besadii
Reviewed-by: Florian Klink <flokli@flokli.de>

snix/Cargo.lock

@@ -4294,6 +4294,7 @@ dependencies = [
  "tokio",
  "tracing",
  "tracing-indicatif",
+ "url",
  "wu-manber",
 ]
 

snix/Cargo.nix

@@ -14114,6 +14114,10 @@ rec {
             name = "tracing-indicatif";
             packageId = "tracing-indicatif";
           }
+          {
+            name = "url";
+            packageId = "url";
+          }
           {
             name = "wu-manber";
             packageId = "wu-manber";

snix/cli/Cargo.toml

@@ -27,6 +27,7 @@ tracing-indicatif.workspace = true
 rustc-hash.workspace = true
 mimalloc.workspace = true
 wu-manber.workspace = true
+url.workspace = true
 
 [dev-dependencies]
 expect-test.workspace = true

snix/cli/src/args.rs

@@ -2,6 +2,7 @@ use std::path::PathBuf;
 
 use clap::Parser;
 use snix_store::utils::ServiceUrlsMemory;
+use url::Url;
 
 /// Provides a CLI interface to trigger evaluation using snix-eval.
 ///
@@ -79,6 +80,13 @@ pub struct Args {
     /// Snix does not read from these.
     #[clap(long)]
     pub drv_dumpdir: Option<PathBuf>,
+
+    /// A list of web servers used by builtins.fetchurl to obtain files by hash.
+    /// Given a hash algorithm ha and a base-16 hash h, Nix will try to download the file
+    /// from hashed-mirror/ha/h. This allows files to be downloaded even if they have
+    /// disappeared from their original URI.
+    #[clap(long, default_values_t = [Url::parse("https://tarballs.nixos.org/").unwrap()])]
+    pub hashed_mirrors: Vec<Url>,
 }
 
 impl Args {

snix/cli/src/lib.rs

@@ -56,6 +56,7 @@ pub fn init_io_handle(tokio_runtime: &tokio::runtime::Runtime, args: &Args) -> R
         nar_calculation_service.into(),
         build_service.into(),
         tokio_runtime.handle().clone(),
+        args.hashed_mirrors.clone(),
     ))
 }
 

snix/glue/benches/eval.rs

@@ -38,6 +38,7 @@ fn interpret(code: &str) {
         nar_calculation_service.into(),
         Arc::<DummyBuildService>::default(),
         TOKIO_RUNTIME.handle().clone(),
+        Vec::new(),
     ));
 
     let mut eval_builder = snix_eval::Evaluation::builder(Box::new(SnixIO::new(

snix/glue/src/builtins/mod.rs

@@ -91,6 +91,7 @@ mod tests {
             nar_calculation_service.into(),
             Arc::<DummyBuildService>::default(),
             runtime.handle().clone(),
+            Vec::new(),
         ));
 
         let mut eval_builder = snix_eval::Evaluation::builder(io.clone() as Rc<dyn EvalIO>);

snix/glue/src/fetchers/mod.rs

@@ -176,6 +176,7 @@ pub struct Fetcher<BS, DS, PS, NS> {
     directory_service: DS,
     path_info_service: PS,
     nar_calculation_service: NS,
+    _hashed_mirrors: Vec<Url>,
 }
 
 impl<BS, DS, PS, NS> Fetcher<BS, DS, PS, NS> {
@@ -184,6 +185,7 @@ impl<BS, DS, PS, NS> Fetcher<BS, DS, PS, NS> {
         directory_service: DS,
         path_info_service: PS,
         nar_calculation_service: NS,
+        hashed_mirrors: Vec<Url>,
     ) -> Self {
         Self {
             http_client: reqwest::Client::builder()
@@ -194,6 +196,7 @@ impl<BS, DS, PS, NS> Fetcher<BS, DS, PS, NS> {
             directory_service,
             path_info_service,
             nar_calculation_service,
+            _hashed_mirrors: hashed_mirrors,
         }
     }
 

snix/glue/src/snix_store_io.rs

@@ -13,6 +13,7 @@ use std::{
 use tokio_util::io::SyncIoBridge;
 use tracing::{Level, Span, error, instrument, warn};
 use tracing_indicatif::span_ext::IndicatifSpanExt;
+use url::Url;
 
 use snix_castore::{
     Node,
@@ -72,6 +73,7 @@ impl SnixStoreIO {
         nar_calculation_service: Arc<dyn NarCalculationService>,
         build_service: Arc<dyn BuildService>,
         tokio_handle: tokio::runtime::Handle,
+        hashed_mirrors: Vec<Url>,
     ) -> Self {
         Self {
             blob_service: blob_service.clone(),
@@ -86,6 +88,7 @@ impl SnixStoreIO {
                 directory_service,
                 path_info_service,
                 nar_calculation_service,
+                hashed_mirrors,
             ),
             known_paths: Default::default(),
         }
@@ -516,6 +519,7 @@ mod tests {
             nar_calculation_service.into(),
             Arc::<DummyBuildService>::default(),
             tokio_runtime.handle().clone(),
+            Vec::new(),
         ));
 
         let mut eval_builder =

snix/glue/src/tests/mod.rs

@@ -48,6 +48,7 @@ fn eval_test(code_path: PathBuf, expect_success: bool) {
         nar_calculation_service.into(),
         Arc::new(DummyBuildService::default()),
         tokio_runtime.handle().clone(),
+        Vec::new(),
     ));
     // Wrap with SnixIO, so <nix/fetchurl.nix can be imported.
     let mut eval_builder = snix_eval::Evaluation::builder(Box::new(SnixIO::new(