feat(nix-compat/nix_http): init parse_nar[info]_str

This moves the URL component parsing code we had in nar-bridge to
nix-compat.

We change the function signature to return an Option, not a
Result<_, StatusCode>.

This allows returning more appropriate error codes, as we can
ok_or(…) at the callsite, which we now do: on an upload to an
invalid path, we now return "unauthorized", while on a GET/HEAD, we
return "not found".

This also adds support to parse compression suffixes. While not
supported in nar-bridge, other users of nix-compat might very well want
to parse these paths.

Also fix the error message when parsing NAR urls, it mentioned 32, not
52, which is a copypasta error from the narinfo URL parsing code.

Change-Id: Id1be9a8044814b54ce68b125c52dfe933c9c4f74
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12260
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Autosubmit: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
Florian Klink 2024-08-21 11:06:12 +03:00 committed by clbot
parent 2357079891
commit e03ea11bad
15 changed files with 634 additions and 178 deletions

1
tvix/Cargo.lock generated
View file

@ -2361,6 +2361,7 @@ dependencies = [
"thiserror",
"tokio",
"tokio-test",
"tracing",
"zstd",
]

View file

@ -7443,6 +7443,10 @@ rec {
optional = true;
features = [ "io-util" "macros" ];
}
{
name = "tracing";
packageId = "tracing";
}
];
devDependencies = [
{

View file

@ -5,7 +5,7 @@ use axum::response::Response;
use bytes::Bytes;
use data_encoding::BASE64URL_NOPAD;
use futures::TryStreamExt;
use nix_compat::nixbase32;
use nix_compat::{nix_http, nixbase32};
use serde::Deserialize;
use std::io;
use tokio_util::io::ReaderStream;
@ -92,7 +92,14 @@ pub async fn put(
}): axum::extract::State<AppState>,
request: axum::extract::Request,
) -> Result<&'static str, StatusCode> {
let nar_hash_expected = parse_nar_str(&nar_str)?;
let (nar_hash_expected, compression_suffix) =
nix_http::parse_nar_str(&nar_str).ok_or(StatusCode::UNAUTHORIZED)?;
// No paths with compression suffix are supported.
if !compression_suffix.is_empty() {
warn!(%compression_suffix, "invalid compression suffix requested");
return Err(StatusCode::UNAUTHORIZED);
}
let s = request.into_body().into_data_stream();
@ -133,54 +140,3 @@ pub async fn put(
// FUTUREWORK: maybe head by narhash. Though not too critical, as we do
// implement HEAD for .narinfo.
/// Parses a `14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar`
/// string and returns the nixbase32-decoded digest.
/// No compression is supported.
fn parse_nar_str(s: &str) -> Result<[u8; 32], StatusCode> {
if !s.is_char_boundary(52) {
warn!("invalid string, no char boundary at 32");
return Err(StatusCode::NOT_FOUND);
}
Ok(match s.split_at(52) {
(hash_str, ".nar") => {
// we know this is 52 bytes
let hash_str_fixed: [u8; 52] = hash_str.as_bytes().try_into().unwrap();
nixbase32::decode_fixed(hash_str_fixed).map_err(|e| {
warn!(err=%e, "invalid digest");
StatusCode::NOT_FOUND
})?
}
_ => {
warn!("invalid string");
return Err(StatusCode::BAD_REQUEST);
}
})
}
#[cfg(test)]
mod test {
use super::parse_nar_str;
use hex_literal::hex;
#[test]
fn success() {
assert_eq!(
hex!("13a8cf7ca57f68a9f1752acee36a72a55187d3a954443c112818926f26109d91"),
parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar").unwrap()
)
}
#[test]
fn failure() {
assert!(
parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar.x").is_err()
);
assert!(
parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar.xz").is_err()
);
assert!(parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0").is_err());
assert!(parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0🦊.nar").is_err())
}
}

View file

@ -1,6 +1,6 @@
use axum::http::StatusCode;
use bytes::Bytes;
use nix_compat::{narinfo::NarInfo, nixbase32};
use nix_compat::{narinfo::NarInfo, nix_http, nixbase32};
use prost::Message;
use tracing::{instrument, warn, Span};
use tvix_castore::proto::{self as castorepb};
@ -18,7 +18,7 @@ pub async fn head(
path_info_service, ..
}): axum::extract::State<AppState>,
) -> Result<&'static str, StatusCode> {
let digest = parse_narinfo_str(&narinfo_str)?;
let digest = nix_http::parse_narinfo_str(&narinfo_str).ok_or(StatusCode::NOT_FOUND)?;
Span::current().record("path_info.digest", &narinfo_str[0..32]);
if path_info_service
@ -44,7 +44,7 @@ pub async fn get(
path_info_service, ..
}): axum::extract::State<AppState>,
) -> Result<String, StatusCode> {
let digest = parse_narinfo_str(&narinfo_str)?;
let digest = nix_http::parse_narinfo_str(&narinfo_str).ok_or(StatusCode::NOT_FOUND)?;
Span::current().record("path_info.digest", &narinfo_str[0..32]);
// fetch the PathInfo
@ -101,7 +101,7 @@ pub async fn put(
}): axum::extract::State<AppState>,
request: axum::extract::Request,
) -> Result<&'static str, StatusCode> {
let _narinfo_digest = parse_narinfo_str(&narinfo_str)?;
let _narinfo_digest = nix_http::parse_narinfo_str(&narinfo_str).ok_or(StatusCode::UNAUTHORIZED);
Span::current().record("path_info.digest", &narinfo_str[0..32]);
let narinfo_bytes: Bytes = axum::body::to_bytes(request.into_body(), NARINFO_LIMIT)
@ -157,49 +157,3 @@ pub async fn put(
}
}
}
/// Parses a `3mzh8lvgbynm9daj7c82k2sfsfhrsfsy.narinfo` string and returns the
/// nixbase32-decoded digest.
fn parse_narinfo_str(s: &str) -> Result<[u8; 20], StatusCode> {
if !s.is_char_boundary(32) {
warn!("invalid string, no char boundary at 32");
return Err(StatusCode::NOT_FOUND);
}
Ok(match s.split_at(32) {
(hash_str, ".narinfo") => {
// we know this is 32 bytes
let hash_str_fixed: [u8; 32] = hash_str.as_bytes().try_into().unwrap();
nixbase32::decode_fixed(hash_str_fixed).map_err(|e| {
warn!(err=%e, "invalid digest");
StatusCode::NOT_FOUND
})?
}
_ => {
warn!("invalid string");
return Err(StatusCode::NOT_FOUND);
}
})
}
#[cfg(test)]
mod test {
use super::parse_narinfo_str;
use hex_literal::hex;
#[test]
fn success() {
assert_eq!(
hex!("8a12321522fd91efbd60ebb2481af88580f61600"),
parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la.narinfo").unwrap()
);
}
#[test]
fn failure() {
assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la").is_err());
assert!(parse_narinfo_str("/00bgd045z0d4icpbc2yyz4gx48ak44la").is_err());
assert!(parse_narinfo_str("000000").is_err());
assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44l🦊.narinfo").is_err());
}
}

View file

@ -27,6 +27,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sha2 = "0.10.6"
thiserror = "1.0.38"
tracing = "0.1.37"
[dependencies.tokio]
optional = true

View file

@ -2,6 +2,7 @@ pub(crate) mod aterm;
pub mod derivation;
pub mod nar;
pub mod narinfo;
pub mod nix_http;
pub mod nixbase32;
pub mod nixcpp;
pub mod nixhash;

View file

@ -0,0 +1,108 @@
use tracing::trace;
use crate::nixbase32;
/// Parses a `14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar`
/// string and returns the nixbase32-decoded digest, as well as the compression
/// suffix (which might be empty).
pub fn parse_nar_str(s: &str) -> Option<([u8; 32], &str)> {
if !s.is_char_boundary(52) {
trace!("invalid string, no char boundary at 52");
return None;
}
let (hash_str, suffix) = s.split_at(52);
// we know hash_str is 52 bytes, so it's ok to unwrap here.
let hash_str_fixed: [u8; 52] = hash_str.as_bytes().try_into().unwrap();
match suffix.strip_prefix(".nar") {
Some(compression_suffix) => match nixbase32::decode_fixed(hash_str_fixed) {
Err(e) => {
trace!(err=%e, "invalid nixbase32 encoding");
None
}
Ok(digest) => Some((digest, compression_suffix)),
},
None => {
trace!("no .nar suffix");
None
}
}
}
/// Parses a `3mzh8lvgbynm9daj7c82k2sfsfhrsfsy.narinfo` string and returns the
/// nixbase32-decoded digest.
pub fn parse_narinfo_str(s: &str) -> Option<[u8; 20]> {
if !s.is_char_boundary(32) {
trace!("invalid string, no char boundary at 32");
return None;
}
match s.split_at(32) {
(hash_str, ".narinfo") => {
// we know this is 32 bytes, so it's ok to unwrap here.
let hash_str_fixed: [u8; 32] = hash_str.as_bytes().try_into().unwrap();
match nixbase32::decode_fixed(hash_str_fixed) {
Err(e) => {
trace!(err=%e, "invalid nixbase32 encoding");
None
}
Ok(digest) => Some(digest),
}
}
_ => {
trace!("invalid string, no .narinfo suffix");
None
}
}
}
#[cfg(test)]
mod test {
use super::{parse_nar_str, parse_narinfo_str};
use hex_literal::hex;
#[test]
fn parse_nar_str_success() {
assert_eq!(
(
hex!("13a8cf7ca57f68a9f1752acee36a72a55187d3a954443c112818926f26109d91"),
""
),
parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar").unwrap()
);
assert_eq!(
(
hex!("13a8cf7ca57f68a9f1752acee36a72a55187d3a954443c112818926f26109d91"),
".xz"
),
parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0k.nar.xz").unwrap()
)
}
#[test]
fn parse_nar_str_failure() {
assert!(parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0").is_none());
assert!(
parse_nar_str("14cx20k6z4hq508kqi2lm79qfld5f9mf7kiafpqsjs3zlmycza0🦊.nar").is_none()
)
}
#[test]
fn parse_narinfo_str_success() {
assert_eq!(
hex!("8a12321522fd91efbd60ebb2481af88580f61600"),
parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la.narinfo").unwrap()
);
}
#[test]
fn parse_narinfo_str_failure() {
assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44la").is_none());
assert!(parse_narinfo_str("/00bgd045z0d4icpbc2yyz4gx48ak44la").is_none());
assert!(parse_narinfo_str("000000").is_none());
assert!(parse_narinfo_str("00bgd045z0d4icpbc2yyz4gx48ak44l🦊.narinfo").is_none());
}
}

View file

@ -1434,6 +1434,7 @@ dependencies = [
"sha2 0.10.8",
"thiserror",
"tokio",
"tracing",
]
[[package]]
@ -2776,9 +2777,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.39",
]
[[package]]
name = "tracing-core"
version = "0.1.32"

View file

@ -4155,6 +4155,10 @@ rec {
optional = true;
features = [ "io-util" "macros" ];
}
{
name = "tracing";
packageId = "tracing";
}
];
devDependencies = [
{
@ -9062,6 +9066,11 @@ rec {
name = "pin-project-lite";
packageId = "pin-project-lite";
}
{
name = "tracing-attributes";
packageId = "tracing-attributes";
optional = true;
}
{
name = "tracing-core";
packageId = "tracing-core";
@ -9077,7 +9086,37 @@ rec {
"tracing-attributes" = [ "dep:tracing-attributes" ];
"valuable" = [ "tracing-core/valuable" ];
};
resolvedDefaultFeatures = [ "std" ];
resolvedDefaultFeatures = [ "attributes" "default" "std" "tracing-attributes" ];
};
"tracing-attributes" = rec {
crateName = "tracing-attributes";
version = "0.1.27";
edition = "2018";
sha256 = "1rvb5dn9z6d0xdj14r403z0af0bbaqhg02hq4jc97g5wds6lqw1l";
procMacro = true;
libName = "tracing_attributes";
authors = [
"Tokio Contributors <team@tokio.rs>"
"Eliza Weisman <eliza@buoyant.io>"
"David Barsky <dbarsky@amazon.com>"
];
dependencies = [
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.39";
usesDefaultFeatures = false;
features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ];
}
];
features = { };
};
"tracing-core" = rec {
crateName = "tracing-core";

View file

@ -970,6 +970,7 @@ dependencies = [
"sha2",
"thiserror",
"tokio",
"tracing",
]
[[package]]
@ -1872,6 +1873,37 @@ dependencies = [
"tokio",
]
[[package]]
name = "tracing"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.39",
]
[[package]]
name = "tracing-core"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
]
[[package]]
name = "typenum"
version = "1.17.0"

View file

@ -2852,6 +2852,10 @@ rec {
optional = true;
features = [ "io-util" "macros" ];
}
{
name = "tracing";
packageId = "tracing";
}
];
devDependencies = [
{
@ -6370,6 +6374,96 @@ rec {
};
resolvedDefaultFeatures = [ "default" "io" "io-util" ];
};
"tracing" = rec {
crateName = "tracing";
version = "0.1.40";
edition = "2018";
sha256 = "1vv48dac9zgj9650pg2b4d0j3w6f3x9gbggf43scq5hrlysklln3";
authors = [
"Eliza Weisman <eliza@buoyant.io>"
"Tokio Contributors <team@tokio.rs>"
];
dependencies = [
{
name = "pin-project-lite";
packageId = "pin-project-lite";
}
{
name = "tracing-attributes";
packageId = "tracing-attributes";
optional = true;
}
{
name = "tracing-core";
packageId = "tracing-core";
usesDefaultFeatures = false;
}
];
features = {
"attributes" = [ "tracing-attributes" ];
"default" = [ "std" "attributes" ];
"log" = [ "dep:log" ];
"log-always" = [ "log" ];
"std" = [ "tracing-core/std" ];
"tracing-attributes" = [ "dep:tracing-attributes" ];
"valuable" = [ "tracing-core/valuable" ];
};
resolvedDefaultFeatures = [ "attributes" "default" "std" "tracing-attributes" ];
};
"tracing-attributes" = rec {
crateName = "tracing-attributes";
version = "0.1.27";
edition = "2018";
sha256 = "1rvb5dn9z6d0xdj14r403z0af0bbaqhg02hq4jc97g5wds6lqw1l";
procMacro = true;
libName = "tracing_attributes";
authors = [
"Tokio Contributors <team@tokio.rs>"
"Eliza Weisman <eliza@buoyant.io>"
"David Barsky <dbarsky@amazon.com>"
];
dependencies = [
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.39";
usesDefaultFeatures = false;
features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ];
}
];
features = { };
};
"tracing-core" = rec {
crateName = "tracing-core";
version = "0.1.32";
edition = "2018";
sha256 = "0m5aglin3cdwxpvbg6kz0r9r0k31j48n0kcfwsp6l49z26k3svf0";
libName = "tracing_core";
authors = [
"Tokio Contributors <team@tokio.rs>"
];
dependencies = [
{
name = "once_cell";
packageId = "once_cell";
optional = true;
}
];
features = {
"default" = [ "std" "valuable/std" ];
"once_cell" = [ "dep:once_cell" ];
"std" = [ "once_cell" ];
"valuable" = [ "dep:valuable" ];
};
resolvedDefaultFeatures = [ "once_cell" "std" ];
};
"typenum" = rec {
crateName = "typenum";
version = "1.17.0";

View file

@ -941,6 +941,7 @@ dependencies = [
"sha2",
"thiserror",
"tokio",
"tracing",
]
[[package]]
@ -1876,6 +1877,37 @@ dependencies = [
"tokio",
]
[[package]]
name = "tracing"
version = "0.1.40"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
dependencies = [
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.48",
]
[[package]]
name = "tracing-core"
version = "0.1.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
dependencies = [
"once_cell",
]
[[package]]
name = "typenum"
version = "1.17.0"

View file

@ -2758,6 +2758,10 @@ rec {
optional = true;
features = [ "io-util" "macros" ];
}
{
name = "tracing";
packageId = "tracing";
}
];
devDependencies = [
{
@ -6321,6 +6325,96 @@ rec {
};
resolvedDefaultFeatures = [ "default" "io" "io-util" ];
};
"tracing" = rec {
crateName = "tracing";
version = "0.1.40";
edition = "2018";
sha256 = "1vv48dac9zgj9650pg2b4d0j3w6f3x9gbggf43scq5hrlysklln3";
authors = [
"Eliza Weisman <eliza@buoyant.io>"
"Tokio Contributors <team@tokio.rs>"
];
dependencies = [
{
name = "pin-project-lite";
packageId = "pin-project-lite";
}
{
name = "tracing-attributes";
packageId = "tracing-attributes";
optional = true;
}
{
name = "tracing-core";
packageId = "tracing-core";
usesDefaultFeatures = false;
}
];
features = {
"attributes" = [ "tracing-attributes" ];
"default" = [ "std" "attributes" ];
"log" = [ "dep:log" ];
"log-always" = [ "log" ];
"std" = [ "tracing-core/std" ];
"tracing-attributes" = [ "dep:tracing-attributes" ];
"valuable" = [ "tracing-core/valuable" ];
};
resolvedDefaultFeatures = [ "attributes" "default" "std" "tracing-attributes" ];
};
"tracing-attributes" = rec {
crateName = "tracing-attributes";
version = "0.1.27";
edition = "2018";
sha256 = "1rvb5dn9z6d0xdj14r403z0af0bbaqhg02hq4jc97g5wds6lqw1l";
procMacro = true;
libName = "tracing_attributes";
authors = [
"Tokio Contributors <team@tokio.rs>"
"Eliza Weisman <eliza@buoyant.io>"
"David Barsky <dbarsky@amazon.com>"
];
dependencies = [
{
name = "proc-macro2";
packageId = "proc-macro2";
}
{
name = "quote";
packageId = "quote";
}
{
name = "syn";
packageId = "syn 2.0.48";
usesDefaultFeatures = false;
features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ];
}
];
features = { };
};
"tracing-core" = rec {
crateName = "tracing-core";
version = "0.1.32";
edition = "2018";
sha256 = "0m5aglin3cdwxpvbg6kz0r9r0k31j48n0kcfwsp6l49z26k3svf0";
libName = "tracing_core";
authors = [
"Tokio Contributors <team@tokio.rs>"
];
dependencies = [
{
name = "once_cell";
packageId = "once_cell";
optional = true;
}
];
features = {
"default" = [ "std" "valuable/std" ];
"once_cell" = [ "dep:once_cell" ];
"std" = [ "once_cell" ];
"valuable" = [ "dep:valuable" ];
};
resolvedDefaultFeatures = [ "once_cell" "std" ];
};
"typenum" = rec {
crateName = "typenum";
version = "1.17.0";