From d741ca4bb13865fa3eed7056bbee05b82243110f Mon Sep 17 00:00:00 2001 From: Vova Kryachko Date: Wed, 11 Jun 2025 04:59:50 -0400 Subject: [PATCH] feat(glue): Add hashed_mirrors support to eval fetcher This change adds basic scaffolding to allow configuring hashed_mirrors that will be used by fetchurl to download artifacts by their hash, this is useful in case certain URLs are no longer available but required to bootstrap nixpkgs stdenv. These urls will have higher priority than the url specified in fetchurl(and friends) and will be attempted before falling back to the actual url specified in fetchurl. Change-Id: I589bdef609075f274cbdf6b26af602cafaa7496a Reviewed-on: https://cl.snix.dev/c/snix/+/30567 Tested-by: besadii Reviewed-by: Florian Klink --- snix/Cargo.lock | 1 + snix/Cargo.nix | 4 ++++ snix/cli/Cargo.toml | 1 + snix/cli/src/args.rs | 8 ++++++++ snix/cli/src/lib.rs | 1 + snix/glue/benches/eval.rs | 1 + snix/glue/src/builtins/mod.rs | 1 + snix/glue/src/fetchers/mod.rs | 3 +++ snix/glue/src/snix_store_io.rs | 4 ++++ snix/glue/src/tests/mod.rs | 1 + 10 files changed, 25 insertions(+) diff --git a/snix/Cargo.lock b/snix/Cargo.lock index b1f7dbb1d..ad09cbbdb 100644 --- a/snix/Cargo.lock +++ b/snix/Cargo.lock @@ -4294,6 +4294,7 @@ dependencies = [ "tokio", "tracing", "tracing-indicatif", + "url", "wu-manber", ] diff --git a/snix/Cargo.nix b/snix/Cargo.nix index d3c65be17..537f1e084 100644 --- a/snix/Cargo.nix +++ b/snix/Cargo.nix @@ -14114,6 +14114,10 @@ rec { name = "tracing-indicatif"; packageId = "tracing-indicatif"; } + { + name = "url"; + packageId = "url"; + } { name = "wu-manber"; packageId = "wu-manber"; diff --git a/snix/cli/Cargo.toml b/snix/cli/Cargo.toml index 4672a788e..2af880ff3 100644 --- a/snix/cli/Cargo.toml +++ b/snix/cli/Cargo.toml @@ -27,6 +27,7 @@ tracing-indicatif.workspace = true rustc-hash.workspace = true mimalloc.workspace = true wu-manber.workspace = true +url.workspace = true [dev-dependencies] expect-test.workspace = true diff --git a/snix/cli/src/args.rs b/snix/cli/src/args.rs index 9ddc4bdec..bae585ee3 100644 --- a/snix/cli/src/args.rs +++ b/snix/cli/src/args.rs @@ -2,6 +2,7 @@ use std::path::PathBuf; use clap::Parser; use snix_store::utils::ServiceUrlsMemory; +use url::Url; /// Provides a CLI interface to trigger evaluation using snix-eval. /// @@ -79,6 +80,13 @@ pub struct Args { /// Snix does not read from these. #[clap(long)] pub drv_dumpdir: Option, + + /// A list of web servers used by builtins.fetchurl to obtain files by hash. + /// Given a hash algorithm ha and a base-16 hash h, Nix will try to download the file + /// from hashed-mirror/ha/h. This allows files to be downloaded even if they have + /// disappeared from their original URI. + #[clap(long, default_values_t = [Url::parse("https://tarballs.nixos.org/").unwrap()])] + pub hashed_mirrors: Vec, } impl Args { diff --git a/snix/cli/src/lib.rs b/snix/cli/src/lib.rs index 2fdeeed3b..1c8035bd1 100644 --- a/snix/cli/src/lib.rs +++ b/snix/cli/src/lib.rs @@ -56,6 +56,7 @@ pub fn init_io_handle(tokio_runtime: &tokio::runtime::Runtime, args: &Args) -> R nar_calculation_service.into(), build_service.into(), tokio_runtime.handle().clone(), + args.hashed_mirrors.clone(), )) } diff --git a/snix/glue/benches/eval.rs b/snix/glue/benches/eval.rs index 03baa045b..d2a735b35 100644 --- a/snix/glue/benches/eval.rs +++ b/snix/glue/benches/eval.rs @@ -38,6 +38,7 @@ fn interpret(code: &str) { nar_calculation_service.into(), Arc::::default(), TOKIO_RUNTIME.handle().clone(), + Vec::new(), )); let mut eval_builder = snix_eval::Evaluation::builder(Box::new(SnixIO::new( diff --git a/snix/glue/src/builtins/mod.rs b/snix/glue/src/builtins/mod.rs index e0a8155ec..6704d5bda 100644 --- a/snix/glue/src/builtins/mod.rs +++ b/snix/glue/src/builtins/mod.rs @@ -91,6 +91,7 @@ mod tests { nar_calculation_service.into(), Arc::::default(), runtime.handle().clone(), + Vec::new(), )); let mut eval_builder = snix_eval::Evaluation::builder(io.clone() as Rc); diff --git a/snix/glue/src/fetchers/mod.rs b/snix/glue/src/fetchers/mod.rs index 0814a8efd..97a667c74 100644 --- a/snix/glue/src/fetchers/mod.rs +++ b/snix/glue/src/fetchers/mod.rs @@ -176,6 +176,7 @@ pub struct Fetcher { directory_service: DS, path_info_service: PS, nar_calculation_service: NS, + _hashed_mirrors: Vec, } impl Fetcher { @@ -184,6 +185,7 @@ impl Fetcher { directory_service: DS, path_info_service: PS, nar_calculation_service: NS, + hashed_mirrors: Vec, ) -> Self { Self { http_client: reqwest::Client::builder() @@ -194,6 +196,7 @@ impl Fetcher { directory_service, path_info_service, nar_calculation_service, + _hashed_mirrors: hashed_mirrors, } } diff --git a/snix/glue/src/snix_store_io.rs b/snix/glue/src/snix_store_io.rs index 33f022b7d..85890d34d 100644 --- a/snix/glue/src/snix_store_io.rs +++ b/snix/glue/src/snix_store_io.rs @@ -13,6 +13,7 @@ use std::{ use tokio_util::io::SyncIoBridge; use tracing::{Level, Span, error, instrument, warn}; use tracing_indicatif::span_ext::IndicatifSpanExt; +use url::Url; use snix_castore::{ Node, @@ -72,6 +73,7 @@ impl SnixStoreIO { nar_calculation_service: Arc, build_service: Arc, tokio_handle: tokio::runtime::Handle, + hashed_mirrors: Vec, ) -> Self { Self { blob_service: blob_service.clone(), @@ -86,6 +88,7 @@ impl SnixStoreIO { directory_service, path_info_service, nar_calculation_service, + hashed_mirrors, ), known_paths: Default::default(), } @@ -516,6 +519,7 @@ mod tests { nar_calculation_service.into(), Arc::::default(), tokio_runtime.handle().clone(), + Vec::new(), )); let mut eval_builder = diff --git a/snix/glue/src/tests/mod.rs b/snix/glue/src/tests/mod.rs index fffacd648..d0f9b6e08 100644 --- a/snix/glue/src/tests/mod.rs +++ b/snix/glue/src/tests/mod.rs @@ -48,6 +48,7 @@ fn eval_test(code_path: PathBuf, expect_success: bool) { nar_calculation_service.into(), Arc::new(DummyBuildService::default()), tokio_runtime.handle().clone(), + Vec::new(), )); // Wrap with SnixIO, so