From 6b48bcc1bf86ef4065fd10d3ea9859e1289bb4de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marijan=20Petri=C4=8Devi=C4=87?= Date: Fri, 21 Mar 2025 22:34:19 -0700 Subject: [PATCH] feat(snix/castore-http): initial implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The castore-http crate provides both a binary and a library interface to serve a single castore root node over HTTP. The library function `get_root_node_contents` will return a `axum::Response` for a requested path in the castore root node depending on the requested paths type. If the requested path in the root node is a directory, we return: - a index file if there is a file matching one of the configurable `index_names` - a directory listing, if no `index_names` were configured and `auto_index` was enabled - the FORBIDDEN status code if no `index_names` were set nor `auto_index` was enabled If the requested path in the root node is a file, we return the file. If the requested path in the root node is a symlink, we figure out wether the target exists and return a REDIRECT. If the requested path doesn't exist in the root node, we respond with NOT_FOUND The binary wraps this functionality and allows one to specify the desired root node by providing its base-64 encoded representation as well as the other configuration parameters affecting the behavior of `get_root_node_contents`. Change-Id: I737482299f788ec0244c54b52042f9eb655a05c2 Reviewed-on: https://cl.snix.dev/c/snix/+/30245 Autosubmit: Marijan Petričević Reviewed-by: Marijan Petričević Tested-by: besadii Reviewed-by: Florian Klink --- snix/Cargo.lock | 27 ++ snix/Cargo.nix | 123 +++++++- snix/Cargo.toml | 1 + snix/castore-http/Cargo.toml | 28 ++ snix/castore-http/default.nix | 5 + snix/castore-http/src/app_state.rs | 13 + snix/castore-http/src/cli.rs | 34 ++ snix/castore-http/src/lib.rs | 266 ++++++++++++++++ snix/castore-http/src/main.rs | 47 +++ snix/castore-http/src/router.rs | 62 ++++ snix/castore-http/src/routes.rs | 484 +++++++++++++++++++++++++++++ snix/utils.nix | 4 + 12 files changed, 1093 insertions(+), 1 deletion(-) create mode 100644 snix/castore-http/Cargo.toml create mode 100644 snix/castore-http/default.nix create mode 100644 snix/castore-http/src/app_state.rs create mode 100644 snix/castore-http/src/cli.rs create mode 100644 snix/castore-http/src/lib.rs create mode 100644 snix/castore-http/src/main.rs create mode 100644 snix/castore-http/src/router.rs create mode 100644 snix/castore-http/src/routes.rs diff --git a/snix/Cargo.lock b/snix/Cargo.lock index c17db5a44..e15738ea1 100644 --- a/snix/Cargo.lock +++ b/snix/Cargo.lock @@ -4248,6 +4248,32 @@ dependencies = [ "zstd", ] +[[package]] +name = "snix-castore-http" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "axum-extra", + "axum-range", + "axum-test", + "blake3", + "bytes", + "clap", + "data-encoding", + "mime", + "mime_guess", + "path-clean", + "prost", + "snix-castore", + "tokio", + "tokio-listener", + "tokio-util", + "tracing", + "tracing-subscriber", + "tracing-test", +] + [[package]] name = "snix-cli" version = "0.1.0" @@ -4800,6 +4826,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.52.0", ] diff --git a/snix/Cargo.nix b/snix/Cargo.nix index 9f2c25c9d..529cce9e9 100644 --- a/snix/Cargo.nix +++ b/snix/Cargo.nix @@ -105,6 +105,16 @@ rec { # File a bug if you depend on any for non-debug work! debug = internal.debugCrate { inherit packageId; }; }; + "snix-castore-http" = rec { + packageId = "snix-castore-http"; + build = internal.buildRustCrateWithFeatures { + packageId = "snix-castore-http"; + }; + + # Debug support which might change between releases. + # File a bug if you depend on any for non-debug work! + debug = internal.debugCrate { inherit packageId; }; + }; "snix-cli" = rec { packageId = "snix-cli"; build = internal.buildRustCrateWithFeatures { @@ -13925,6 +13935,109 @@ rec { }; resolvedDefaultFeatures = [ "cloud" "default" "fs" "fuse" "integration" "toml" "tonic-reflection" "virtiofs" "xp-composition-cli" "xp-composition-url-refs" ]; }; + "snix-castore-http" = rec { + crateName = "snix-castore-http"; + version = "0.1.0"; + edition = "2021"; + crateBin = [ + { + name = "snix-castore-http"; + path = "src/main.rs"; + requiredFeatures = [ ]; + } + ]; + src = lib.cleanSourceWith { filter = sourceFilter; src = ./castore-http; }; + libName = "snix_castore_http"; + dependencies = [ + { + name = "anyhow"; + packageId = "anyhow"; + } + { + name = "axum"; + packageId = "axum"; + features = [ "tracing" ]; + } + { + name = "axum-extra"; + packageId = "axum-extra"; + } + { + name = "axum-range"; + packageId = "axum-range"; + } + { + name = "bytes"; + packageId = "bytes"; + } + { + name = "clap"; + packageId = "clap"; + features = [ "derive" ]; + } + { + name = "data-encoding"; + packageId = "data-encoding"; + } + { + name = "mime"; + packageId = "mime"; + } + { + name = "mime_guess"; + packageId = "mime_guess"; + } + { + name = "path-clean"; + packageId = "path-clean"; + } + { + name = "prost"; + packageId = "prost"; + } + { + name = "snix-castore"; + packageId = "snix-castore"; + } + { + name = "tokio"; + packageId = "tokio"; + features = [ "tracing" ]; + } + { + name = "tokio-listener"; + packageId = "tokio-listener"; + features = [ "axum07" "clap" "multi-listener" "sd_listen" ]; + } + { + name = "tokio-util"; + packageId = "tokio-util"; + } + { + name = "tracing"; + packageId = "tracing"; + } + { + name = "tracing-subscriber"; + packageId = "tracing-subscriber"; + } + ]; + devDependencies = [ + { + name = "axum-test"; + packageId = "axum-test"; + } + { + name = "blake3"; + packageId = "blake3"; + } + { + name = "tracing-test"; + packageId = "tracing-test"; + } + ]; + + }; "snix-cli" = rec { crateName = "snix-cli"; version = "0.1.0"; @@ -15703,6 +15816,14 @@ rec { packageId = "tokio-macros"; optional = true; } + { + name = "tracing"; + packageId = "tracing"; + optional = true; + usesDefaultFeatures = false; + target = { target, features }: (target."tokio_unstable" or false); + features = [ "std" ]; + } { name = "windows-sys"; packageId = "windows-sys 0.52.0"; @@ -15747,7 +15868,7 @@ rec { "tracing" = [ "dep:tracing" ]; "windows-sys" = [ "dep:windows-sys" ]; }; - resolvedDefaultFeatures = [ "bytes" "default" "fs" "io-std" "io-util" "libc" "macros" "mio" "net" "process" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "test-util" "time" "tokio-macros" "windows-sys" ]; + resolvedDefaultFeatures = [ "bytes" "default" "fs" "io-std" "io-util" "libc" "macros" "mio" "net" "process" "rt" "rt-multi-thread" "signal" "signal-hook-registry" "socket2" "sync" "test-util" "time" "tokio-macros" "tracing" "windows-sys" ]; }; "tokio-listener" = rec { crateName = "tokio-listener"; diff --git a/snix/Cargo.toml b/snix/Cargo.toml index 9d6d0913e..ac24a2737 100644 --- a/snix/Cargo.toml +++ b/snix/Cargo.toml @@ -21,6 +21,7 @@ resolver = "2" members = [ "build", "castore", + "castore-http", "cli", "eval", "eval/builtin-macros", diff --git a/snix/castore-http/Cargo.toml b/snix/castore-http/Cargo.toml new file mode 100644 index 000000000..62a778bf3 --- /dev/null +++ b/snix/castore-http/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "snix-castore-http" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow.workspace = true +axum = { workspace = true, features = ["tracing"] } +axum-extra.workspace = true +axum-range.workspace = true +bytes.workspace = true +clap = { workspace = true, features = ["derive"] } +data-encoding.workspace = true +mime_guess = "2.0.5" +mime = "0.3.17" +path-clean.workspace = true +prost.workspace = true +tokio = { workspace = true, features = [ "tracing"] } +tokio-listener = { workspace = true, features = ["axum07", "clap", "multi-listener", "sd_listen"] } +tracing.workspace = true +tracing-subscriber.workspace = true +tokio-util.workspace = true +snix-castore = { path = "../castore" } + +[dev-dependencies] +axum-test = "16.4.0" +blake3.workspace = true +tracing-test.workspace = true diff --git a/snix/castore-http/default.nix b/snix/castore-http/default.nix new file mode 100644 index 000000000..fee5c93f8 --- /dev/null +++ b/snix/castore-http/default.nix @@ -0,0 +1,5 @@ +{ depot, ... }: + +(depot.snix.crates.workspaceMembers.snix-castore-http.build.override { + runTests = true; +}) diff --git a/snix/castore-http/src/app_state.rs b/snix/castore-http/src/app_state.rs new file mode 100644 index 000000000..403a4f2ea --- /dev/null +++ b/snix/castore-http/src/app_state.rs @@ -0,0 +1,13 @@ +use snix_castore::{blobservice::BlobService, directoryservice::DirectoryService, Node}; + +use std::sync::Arc; + +pub type AppState = Arc; + +pub struct AppConfig { + pub blob_service: Arc, + pub directory_service: Arc, + pub root_node: Node, + pub index_names: Vec, + pub auto_index: bool, +} diff --git a/snix/castore-http/src/cli.rs b/snix/castore-http/src/cli.rs new file mode 100644 index 000000000..49572c031 --- /dev/null +++ b/snix/castore-http/src/cli.rs @@ -0,0 +1,34 @@ +use clap::Parser; +use snix_castore::utils::ServiceUrlsGrpc; + +#[derive(Parser)] +#[command(author, version, about)] +pub struct CliArgs { + /// The address to listen on + #[clap(flatten)] + pub listen_args: tokio_listener::ListenerAddressLFlag, + // The castore services addresses + #[clap(flatten)] + pub service_addrs: ServiceUrlsGrpc, + /// The castore root node to serve, URL-safe base64-encoded + #[arg( + short, + long, + help = "The castore root node to serve, URL-safe base64-encoded" + )] + pub root_node: String, + /// The name of the file to serve if a client requests a directory e.g. index.html index.htm + #[arg( + short, + long, + help = "The name of the file to serve if a client requests a directory e.g. index.html index.htm" + )] + pub index_names: Vec, + /// Whether a directory listing should be returned if a client requests a directory but none of the `index_names` matched + #[arg( + short, + long, + help = "Whether a directory listing should be returned if a client requests a directory but none of the `index_names` matched" + )] + pub auto_index: bool, +} diff --git a/snix/castore-http/src/lib.rs b/snix/castore-http/src/lib.rs new file mode 100644 index 000000000..8873bde39 --- /dev/null +++ b/snix/castore-http/src/lib.rs @@ -0,0 +1,266 @@ +pub mod app_state; +pub mod cli; +pub mod router; +pub mod routes; + +use std::path; + +use snix_castore::{ + blobservice::BlobService, + directoryservice::{descend_to, DirectoryService}, + B3Digest, Directory, Node, Path, SymlinkTarget, +}; + +use axum::{ + body::Body, + http::{header, StatusCode}, + response::{AppendHeaders, IntoResponse, Redirect, Response}, +}; +use axum_extra::{headers::Range, response::Html, TypedHeader}; +use axum_range::{KnownSize, Ranged}; +use path_clean::PathClean; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use tokio_util::io::ReaderStream; +use tracing::{debug, error, instrument, warn}; + +/// Helper function, descending from the given `root_node` to the `requested_path` specified. +/// Returns HTTP Responses or Status Codes. +/// If the path points to a regular file, it serves its contents. +/// If the path points to a symlink, it sends a redirect to the target (pretending `base_path`, if relative) +/// If the path points to a directory, files of `index_names` are tried, +/// if no files matched then a directory listing is returned if `auto_index` is enabled. +/// +/// Uses the passed [BlobService] and [DirectoryService] +#[allow(clippy::too_many_arguments)] +#[instrument(level = "trace", skip_all, fields(base_path, requested_path), err)] +pub async fn get_root_node_contents>( + blob_service: BS, + directory_service: DS, + base_path: &path::Path, + root_node: &Node, + requested_path: &Path, + range_header: Option>, + index_names: &[S], + auto_index: bool, +) -> Result { + match root_node { + Node::Directory { .. } => { + let requested_node = descend_to(&directory_service, root_node.clone(), requested_path) + .await + .map_err(|err| { + error!(err=%err, "an error occured descending"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or_else(|| { + error!("requested path doesn't exist"); + StatusCode::NOT_FOUND + })?; + match requested_node { + Node::Directory { digest, .. } => { + let requested_directory = directory_service + .get(&digest) + .await + .map_err(|err| { + error!(err=%err, "an error occured getting the directory"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or_else(|| { + error!("directory doesn't exist"); + StatusCode::NOT_FOUND + })?; + + // If there was one or more index configured, try to find it + // in the directory requested by the client, by comparing the bytes + // of each directories immediate child's path with the bytes of the + // configured index name + for index_name in index_names { + if let Some((found_index_file_path, found_index_node)) = requested_directory + .nodes() + .find(|(path, _node)| index_name.as_ref().as_bytes() == path.as_ref()) + { + match found_index_node { + Node::File { digest, size, .. } => { + let found_index_file_path = found_index_file_path.to_string(); + let found_index_file_path = path::Path::new(OsStr::from_bytes( + found_index_file_path.as_bytes(), + )); + return respond_file( + blob_service, + Some(found_index_file_path), + range_header, + digest, + *size, + ) + .await; + } + _ => { + debug!( + path = %found_index_file_path, + "One of the configured index names matched with a + node located in the root node's directory which is + not a file" + ); + } + } + } + } + if auto_index { + return respond_directory_list(&requested_directory, requested_path).await; + } + Err(StatusCode::FORBIDDEN) + } + Node::File { digest, size, .. } => { + let requested_path = + path::Path::new(OsStr::from_bytes(requested_path.as_bytes())); + respond_file( + blob_service, + Some(requested_path), + range_header, + &digest, + size, + ) + .await + } + Node::Symlink { target } => { + let requested_path = + path::Path::new(OsStr::from_bytes(requested_path.as_bytes())); + respond_symlink(base_path, &target, Some(requested_path)).await + } + } + } + Node::File { digest, size, .. } => { + if requested_path.to_string() == "" { + respond_file(blob_service, None, range_header, digest, *size).await + } else { + warn!( + "The client requested a path but the configured root + node being served is a file" + ); + Err(StatusCode::BAD_REQUEST) + } + } + Node::Symlink { target } => { + if requested_path.to_string() == "" { + respond_symlink(base_path, target, None).await + } else { + warn!( + "The client requested a path but the configured root + node being served is a symlink" + ); + Err(StatusCode::BAD_REQUEST) + } + } + } +} + +#[instrument(level = "trace", skip_all)] +pub async fn respond_symlink( + base_path: &path::Path, + symlink_target: &SymlinkTarget, + requested_path: Option<&path::Path>, +) -> Result { + if symlink_target.as_ref() == b"." { + error!("There was a symlink with target '.'"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + + let symlink_target_path = match std::str::from_utf8(symlink_target.as_ref()) { + Ok(s) => path::Path::new(s), + Err(_) => { + error!("Symlink target contains invalid UTF-8"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + let symlink_target_path = if symlink_target_path.is_absolute() { + symlink_target_path.to_path_buf() + } else if let Some(requested_path) = requested_path { + let requested_path_parent = requested_path.parent().ok_or_else(|| { + error!("failed to retrieve parent path for requested path"); + StatusCode::INTERNAL_SERVER_ERROR + })?; + base_path + .join(requested_path_parent) + .join(symlink_target_path) + } else { + base_path.join(symlink_target_path) + }; + + let symlink_target_path = symlink_target_path.clean(); + + if symlink_target_path.starts_with(path::Component::ParentDir) { + error!("the symlink's target path points to a non-existing path"); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + + let symlink_target_path_str = symlink_target_path.to_str().ok_or(StatusCode::NOT_FOUND)?; + Ok(Redirect::temporary(symlink_target_path_str).into_response()) +} + +#[instrument(level = "trace", skip_all, fields(directory_path, directory))] +pub async fn respond_directory_list( + directory: &Directory, + directory_path: &Path, +) -> Result { + let mut directory_list_html = String::new(); + for (path_component, _node) in directory.nodes() { + let directory_path = directory_path + .try_join(path_component.as_ref()) + .expect("Join path"); + directory_list_html.push_str(&format!( + "
  • {path_component}
    • " + )) + } + Ok(Html(format!( + "{directory_list_html}" + )) + .into_response()) +} + +#[instrument(level = "trace", skip_all, fields(digest, size))] +pub async fn respond_file( + blob_service: BS, + requested_path: Option<&path::Path>, + range_header: Option>, + digest: &B3Digest, + size: u64, +) -> Result { + let blob_reader = blob_service + .open_read(digest) + .await + .map_err(|err| { + error!(err=%err, "failed to read blob"); + StatusCode::INTERNAL_SERVER_ERROR + })? + .ok_or_else(|| { + error!("blob doesn't exist"); + StatusCode::NOT_FOUND + })?; + + let mime_type = requested_path + .and_then(path::Path::extension) + .and_then(std::ffi::OsStr::to_str) + .and_then(|extension| mime_guess::from_ext(extension).first()) + .unwrap_or(mime::APPLICATION_OCTET_STREAM); + match range_header { + None => Ok(( + StatusCode::OK, + AppendHeaders([ + (header::CONTENT_TYPE, mime_type.to_string()), + (header::CONTENT_LENGTH, size.to_string()), + ]), + Body::from_stream(ReaderStream::new(blob_reader)), + ) + .into_response()), + Some(TypedHeader(range)) => Ok(( + StatusCode::OK, + AppendHeaders([ + (header::CONTENT_TYPE, mime_type.to_string()), + (header::CONTENT_LENGTH, size.to_string()), + ]), + Ranged::new(Some(range), KnownSize::sized(blob_reader, size)).into_response(), + ) + .into_response()), + } +} diff --git a/snix/castore-http/src/main.rs b/snix/castore-http/src/main.rs new file mode 100644 index 000000000..3ab52a31d --- /dev/null +++ b/snix/castore-http/src/main.rs @@ -0,0 +1,47 @@ +use snix_castore_http::cli::CliArgs; + +use anyhow::{bail, Context}; +use bytes::Bytes; +use clap::Parser; +use data_encoding::BASE64URL_NOPAD; +use prost::Message; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + tracing_subscriber::fmt().init(); + + let CliArgs { + listen_args, + service_addrs, + root_node, + index_names, + auto_index, + }: CliArgs = snix_castore_http::cli::CliArgs::parse(); + + // b64decode the root node passed *by the user* + let root_entry_proto = BASE64URL_NOPAD + .decode(root_node.as_bytes()) + .context("unable to decode root node b64")?; + + // check the proto size to be somewhat reasonable before parsing it. + if root_entry_proto.len() > 4096 { + bail!("rejected, too large root node"); + } + + // parse the proto + let root_entry: snix_castore::proto::Entry = Message::decode(Bytes::from(root_entry_proto)) + .context("unable to decode root node proto")?; + + let root_node = root_entry + .try_into_anonymous_node() + .context("root node validation failed")?; + + snix_castore_http::router::gen_router( + listen_args, + service_addrs, + root_node, + &index_names, + auto_index, + ) + .await +} diff --git a/snix/castore-http/src/router.rs b/snix/castore-http/src/router.rs new file mode 100644 index 000000000..1d37075b4 --- /dev/null +++ b/snix/castore-http/src/router.rs @@ -0,0 +1,62 @@ +use crate::app_state::{AppConfig, AppState}; +use crate::routes; + +use snix_castore::utils::ServiceUrlsGrpc; +use snix_castore::Node; + +use axum::{routing::get, Router}; +use std::sync::Arc; +use tokio_listener::ListenerAddressLFlag; +use tracing::info; + +/// Runs the snix-castore-http server given the specified CLI arguments +pub async fn gen_router( + listen_args: ListenerAddressLFlag, + service_addrs: ServiceUrlsGrpc, + root_node: Node, + index_names: &[String], + auto_index: bool, +) -> anyhow::Result<()> { + let (blob_service, directory_service) = snix_castore::utils::construct_services(service_addrs) + .await + .expect("failed to construct services"); + + let app_state = Arc::new(AppConfig { + blob_service, + directory_service, + root_node, + index_names: index_names.to_vec(), + auto_index, + }); + + let app = app(app_state); + + let listen_address = &listen_args.listen_address.unwrap_or_else(|| { + "[::]:9000" + .parse() + .expect("invalid fallback listen address") + }); + + let listener = tokio_listener::Listener::bind( + listen_address, + &Default::default(), + &listen_args.listener_options, + ) + .await?; + + info!(listen_address=%listen_address, "starting daemon"); + + tokio_listener::axum07::serve( + listener, + app.into_make_service_with_connect_info::(), + ) + .await?; + Ok(()) +} + +pub fn app(app_state: AppState) -> Router { + Router::new() + .route("/*path", get(routes::root_node_contents)) + .route("/", get(routes::root_node_contents)) + .with_state(app_state) +} diff --git a/snix/castore-http/src/routes.rs b/snix/castore-http/src/routes.rs new file mode 100644 index 000000000..04239540c --- /dev/null +++ b/snix/castore-http/src/routes.rs @@ -0,0 +1,484 @@ +use crate::app_state::AppState; +use crate::get_root_node_contents; + +use snix_castore::PathBuf; + +use axum::{ + extract::{self, State}, + http::StatusCode, + response::Response, +}; +use axum_extra::{headers::Range, TypedHeader}; +use std::path; +use tracing::{debug, instrument}; + +#[instrument(level = "trace", ret, skip_all, fields(maybe_path))] +pub async fn root_node_contents( + maybe_path: Option>, + state: State, + range_header: Option>, +) -> Result { + let requested_path = maybe_path + .map(|extract::Path(path)| PathBuf::from_host_path(path::Path::new(&path), true)) + .transpose() + .map_err(|err| { + debug!(%err, "User requested an invalid path"); + StatusCode::BAD_REQUEST + })?; + let requested_path = match requested_path.as_ref() { + Some(p) => p.as_ref(), + None => &PathBuf::new(), + }; + + get_root_node_contents( + state.blob_service.clone(), + state.directory_service.clone(), + path::Path::new("/"), + &state.root_node, + requested_path, + range_header, + &state.index_names, + state.auto_index, + ) + .await +} + +#[cfg(test)] +mod tests { + use crate::{app_state::AppConfig, router::app}; + + use snix_castore::{ + blobservice::{BlobService, MemoryBlobService}, + directoryservice::{DirectoryService, MemoryDirectoryService}, + fixtures::{DIRECTORY_COMPLICATED, HELLOWORLD_BLOB_CONTENTS, HELLOWORLD_BLOB_DIGEST}, + B3Digest, Directory, Node, + }; + + use axum::http::StatusCode; + use std::io::Cursor; + use std::sync::{Arc, LazyLock}; + use tracing_test::traced_test; + + /// Accepts a root node to be served, and returns a [axum_test::TestServer]. + fn gen_server>( + root_node: Node, + index_names: &[S], + auto_index: bool, + ) -> ( + axum_test::TestServer, + impl BlobService, + impl DirectoryService, + ) { + let blob_service = Arc::new(MemoryBlobService::default()); + let directory_service = Arc::new(MemoryDirectoryService::default()); + + let app = app(Arc::new(AppConfig { + blob_service: blob_service.clone(), + directory_service: directory_service.clone(), + root_node, + index_names: index_names + .iter() + .map(|index| index.as_ref().to_string()) + .collect(), + auto_index, + })); + + ( + axum_test::TestServer::new(app).unwrap(), + blob_service, + directory_service, + ) + } + + pub const INDEX_HTML_BLOB_CONTENTS: &[u8] = + b"Hello World!"; + pub static INDEX_HTML_BLOB_DIGEST: LazyLock = + LazyLock::new(|| blake3::hash(INDEX_HTML_BLOB_CONTENTS).as_bytes().into()); + + pub static DIRECTORY_NESTED_WITH_SYMLINK: LazyLock = LazyLock::new(|| { + Directory::try_from_iter([ + ( + "nested".try_into().unwrap(), + Node::Directory { + digest: DIRECTORY_WITH_SYMLINK.digest(), + size: DIRECTORY_WITH_SYMLINK.size(), + }, + ), + ( + "index.htm".try_into().unwrap(), + Node::File { + digest: INDEX_HTML_BLOB_DIGEST.clone(), + size: INDEX_HTML_BLOB_CONTENTS.len() as u64, + executable: false, + }, + ), + ( + "out_of_base_path_symlink".try_into().unwrap(), + Node::Symlink { + target: "../index.htm".try_into().unwrap(), + }, + ), + ]) + .unwrap() + }); + + pub static DIRECTORY_WITH_SYMLINK: LazyLock = LazyLock::new(|| { + Directory::try_from_iter([ + ( + "index.html".try_into().unwrap(), + Node::File { + digest: INDEX_HTML_BLOB_DIGEST.clone(), + size: INDEX_HTML_BLOB_CONTENTS.len() as u64, + executable: false, + }, + ), + ( + "dot".try_into().unwrap(), + Node::Symlink { + target: ".".try_into().unwrap(), + }, + ), + ( + "symlink".try_into().unwrap(), + Node::Symlink { + target: "index.html".try_into().unwrap(), + }, + ), + ( + "dot_symlink".try_into().unwrap(), + Node::Symlink { + target: "./index.html".try_into().unwrap(), + }, + ), + ( + "dotdot_symlink".try_into().unwrap(), + Node::Symlink { + target: "../index.htm".try_into().unwrap(), + }, + ), + ( + "dotdot_same_symlink".try_into().unwrap(), + Node::Symlink { + target: "../nested/index.html".try_into().unwrap(), + }, + ), + ]) + .unwrap() + }); + + #[traced_test] + #[tokio::test] + async fn test_lists_directory_contents_if_auto_index_enabled() { + let root_node = Node::Directory { + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }; + + // No index but auto-index is enabled + let (server, _blob_service, directory_service) = gen_server::<&str>(root_node, &[], true); + + directory_service + .put(DIRECTORY_COMPLICATED.clone()) + .await + .expect("Failed to insert directory"); + + server + .get("/") + .expect_success() + .await + .assert_text_contains("
  • .keep
  • aa
  • keep
    • "); + } + + #[traced_test] + #[tokio::test] + async fn test_lists_directory_contents_if_auto_index_enabled_for_nested_dir() { + let root_node = Node::Directory { + digest: DIRECTORY_NESTED_WITH_SYMLINK.digest(), + size: DIRECTORY_NESTED_WITH_SYMLINK.size(), + }; + + // No index but auto-index is enabled + let (server, _blob_service, directory_service) = gen_server::<&str>(root_node, &[], true); + let mut directory_service_handle = directory_service.put_multiple_start(); + directory_service_handle + .put(DIRECTORY_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .put(DIRECTORY_NESTED_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .close() + .await + .expect("Failed to close handle"); + + server + .get("/nested") + .expect_success() + .await + .assert_text_contains("
  • dot
  • dot_symlink
  • dotdot_same_symlink
  • dotdot_symlink
  • index.html
  • symlink
    • ") + } + + #[traced_test] + #[tokio::test] + async fn test_responds_index_file_if_configured() { + let root_node = Node::Directory { + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }; + + // .keep is a index file in this test scenario, auto-index is off + let (server, blob_service, directory_service) = + gen_server::<&str>(root_node, &[".keep"], false); + + directory_service + .put(DIRECTORY_COMPLICATED.clone()) + .await + .expect("Failed to insert directory"); + + let mut blob_writer = blob_service.open_write().await; + tokio::io::copy(&mut Cursor::new(vec![]), &mut blob_writer) + .await + .expect("Failed to copy file to BlobWriter"); + blob_writer + .close() + .await + .expect("Failed to close the BlobWriter"); + + server.get("/").expect_success().await; + } + + #[traced_test] + #[tokio::test] + async fn test_responds_index_file_if_configured_in_nested_dir() { + let root_node = Node::Directory { + digest: DIRECTORY_NESTED_WITH_SYMLINK.digest(), + size: DIRECTORY_NESTED_WITH_SYMLINK.size(), + }; + + // .keep is a index file in this test scenario, auto-index is off + let (server, blob_service, directory_service) = + gen_server::<&str>(root_node, &["index.html"], false); + + let mut directory_service_handle = directory_service.put_multiple_start(); + directory_service_handle + .put(DIRECTORY_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .put(DIRECTORY_NESTED_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .close() + .await + .expect("Failed to close handle"); + + let mut blob_writer = blob_service.open_write().await; + tokio::io::copy(&mut Cursor::new(INDEX_HTML_BLOB_CONTENTS), &mut blob_writer) + .await + .expect("Failed to copy file to BlobWriter"); + let digest = blob_writer + .close() + .await + .expect("Failed to close the BlobWriter"); + assert_eq!(digest, *INDEX_HTML_BLOB_DIGEST); + + server.get("/nested").expect_success().await; + } + + #[traced_test] + #[tokio::test] + async fn test_responds_forbidden_if_no_index_configured_nor_auto_index_enabled() { + let root_node = Node::Directory { + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }; + + // no index configured and auto-index disabled + let (server, _blob_service, directory_service) = gen_server::<&str>(root_node, &[], false); + + directory_service + .put(DIRECTORY_COMPLICATED.clone()) + .await + .expect("Failed to insert directory"); + + let response = server.get("/").expect_failure().await; + response.assert_status(StatusCode::FORBIDDEN); + } + + #[traced_test] + #[tokio::test] + async fn test_responds_file() { + let root_node = Node::Directory { + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }; + + let (server, blob_service, directory_service) = gen_server::<&str>(root_node, &[], false); + + directory_service + .put(DIRECTORY_COMPLICATED.clone()) + .await + .expect("Failed to insert directory"); + + let mut blob_writer = blob_service.open_write().await; + tokio::io::copy(&mut Cursor::new(vec![]), &mut blob_writer) + .await + .expect("Failed to copy file to BlobWriter"); + blob_writer + .close() + .await + .expect("Failed to close the BlobWriter"); + + server.get("/.keep").expect_success().await; + } + + #[traced_test] + #[tokio::test] + async fn test_responds_file_and_correct_content_type() { + let root_node = Node::Directory { + digest: DIRECTORY_NESTED_WITH_SYMLINK.digest(), + size: DIRECTORY_NESTED_WITH_SYMLINK.size(), + }; + + let (server, blob_service, directory_service) = gen_server::<&str>(root_node, &[], false); + + let mut directory_service_handle = directory_service.put_multiple_start(); + directory_service_handle + .put(DIRECTORY_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .put(DIRECTORY_NESTED_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .close() + .await + .expect("Failed to close handle"); + + let mut blob_writer = blob_service.open_write().await; + tokio::io::copy(&mut Cursor::new(INDEX_HTML_BLOB_CONTENTS), &mut blob_writer) + .await + .expect("Failed to copy file to BlobWriter"); + let digest = blob_writer + .close() + .await + .expect("Failed to close the BlobWriter"); + assert_eq!(digest, *INDEX_HTML_BLOB_DIGEST); + + let response = server.get("/nested/index.html").expect_success().await; + response.assert_header("Content-Type", "text/html"); + } + + #[traced_test] + #[tokio::test] + async fn test_responds_redirect_if_symlink() { + let root_node = Node::Directory { + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }; + + let (server, _blob_service, directory_service) = gen_server::<&str>(root_node, &[], false); + + directory_service + .put(DIRECTORY_COMPLICATED.clone()) + .await + .expect("Failed to insert directory"); + + let response = server.get("/aa").await; + response.assert_status(StatusCode::TEMPORARY_REDIRECT); + response.assert_header("Location", "/nix/store/somewhereelse"); + } + + #[traced_test] + #[tokio::test] + async fn test_responds_redirect_with_normalized_path_if_symlink() { + let root_node = Node::Directory { + digest: DIRECTORY_NESTED_WITH_SYMLINK.digest(), + size: DIRECTORY_NESTED_WITH_SYMLINK.size(), + }; + + let (server, _blob_service, directory_service) = gen_server::<&str>(root_node, &[], false); + + let mut directory_service_handle = directory_service.put_multiple_start(); + directory_service_handle + .put(DIRECTORY_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .put(DIRECTORY_NESTED_WITH_SYMLINK.clone()) + .await + .expect("Failed to insert directory"); + directory_service_handle + .close() + .await + .expect("Failed to close handle"); + + let response = server.get("/nested/symlink").await; + response.assert_status(StatusCode::TEMPORARY_REDIRECT); + response.assert_header("Location", "/nested/index.html"); + + let response = server.get("/nested/dot_symlink").await; + response.assert_status(StatusCode::TEMPORARY_REDIRECT); + response.assert_header("Location", "/nested/index.html"); + + let response = server.get("/nested/dotdot_symlink").await; + response.assert_status(StatusCode::TEMPORARY_REDIRECT); + response.assert_header("Location", "/index.htm"); + + let response = server.get("/out_of_base_path_symlink").await; + response.assert_status(StatusCode::TEMPORARY_REDIRECT); + response.assert_header("Location", "/index.htm"); + + let response = server.get("/nested/dot").expect_failure().await; + response.assert_status(StatusCode::INTERNAL_SERVER_ERROR); + } + + #[traced_test] + #[tokio::test] + async fn test_returns_bad_request_if_not_valid_path() { + let root_node = Node::Directory { + digest: DIRECTORY_COMPLICATED.digest(), + size: DIRECTORY_COMPLICATED.size(), + }; + + let (server, _blob_service, _directory_service) = gen_server::<&str>(root_node, &[], false); + + // request an invalid path + let response = server.get("//aa").expect_failure().await; + response.assert_status(StatusCode::BAD_REQUEST); + } + + #[traced_test] + #[tokio::test] + async fn test_returns_bad_request_if_root_node_is_file_and_path_requested() { + let root_node = Node::File { + digest: HELLOWORLD_BLOB_DIGEST.clone(), + size: HELLOWORLD_BLOB_CONTENTS.len() as u64, + executable: false, + }; + + let (server, _blob_service, _directory_service) = gen_server::<&str>(root_node, &[], false); + + // request a path while the root node is a file + let response = server.get("/some-path").expect_failure().await; + response.assert_status(StatusCode::BAD_REQUEST); + } + + #[traced_test] + #[tokio::test] + async fn test_returns_bad_request_if_root_node_is_symlink_and_path_requested() { + let root_node = Node::Symlink { + target: "/nix/store/somewhereelse".try_into().unwrap(), + }; + + let (server, _blob_service, _directory_service) = gen_server::<&str>(root_node, &[], false); + + // request a path while the root node is a symlink + let response = server.get("/some-path").expect_failure().await; + response.assert_status(StatusCode::BAD_REQUEST); + } +} diff --git a/snix/utils.nix b/snix/utils.nix index 551bfb42f..a68ceeed9 100644 --- a/snix/utils.nix +++ b/snix/utils.nix @@ -92,6 +92,10 @@ in nativeBuildInputs = [ pkgs.protobuf ]; }; + snix-castore-htp = prev: { + src = filterRustCrateSrc { root = prev.src.origSrc; }; + }; + snix-cli = prev: { src = filterRustCrateSrc rec { root = prev.src.origSrc;