maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(tvix/store/bin): add progress bar infrastructure

Browse source 
This adds the tracing-indicatif crate, and configures it as a layer in
our tracing_subscriber pipeline to emit progress for every span that's
configured so.

It also moves from using std::io::stderr to write logs to using their
writer, to avoid clobbering output.

Progress bar styles are defined in a lazy_static, moving this into a
general tracing is left for later.

This adds some usage of this to the `imports` and `copy` commands.

The output can still be improved a bit - we should  probably split each
task up into a smaller (instrumented) helper functions, so we can create
a progress bar for each task.

Change-Id: I59a1915aa4e0caa89c911632dec59c4cbeba1b89
Reviewed-on: https://cl.tvl.fyi/c/depot/+/11747
Reviewed-by: flokli <flokli@flokli.de>
Reviewed-by: Simon Hauser <simon.hauser@helsinki-systems.de>
Tested-by: BuildkiteCI
Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink 2024-04-17 19:44:07 +03:00 • committed by flokli
parent 9b77ce9f8f
commit 20513e7a52
8 changed files with 452 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

15
tvix/castore/src/import/fs.rs
View file

@ -7,6 +7,8 @@ use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::MetadataExt;
use std::os::unix::fs::PermissionsExt;
use tracing::instrument;
use tracing::Span;
use tracing_indicatif::span_ext::IndicatifSpanExt;
use walkdir::DirEntry;
use walkdir::WalkDir;
@ -37,6 +39,7 @@ where
BS: BlobService + Clone,
DS: DirectoryService,
{
Span::current().pb_start();
let iter = WalkDir::new(path.as_ref())
.follow_links(false)
.follow_root_links(false)
@ -44,7 +47,17 @@ where
.into_iter();
let entries = dir_entries_to_ingestion_stream(blob_service, iter, path.as_ref());
ingest_entries(directory_service, entries).await
ingest_entries(
directory_service,
entries.inspect(|e| {
if let Ok(e) = e {
let s = Span::current();
s.pb_inc(1);
s.pb_set_message(&format!("Ingesting {}", e.path()));
}
}),
)
.await
}
/// Converts an iterator of [walkdir::DirEntry]s into a stream of ingestion entries.

6
tvix/castore/src/import/mod.rs
View file

@ -14,6 +14,8 @@ use crate::proto::FileNode;
use crate::proto::SymlinkNode;
use crate::B3Digest;
use futures::{Stream, StreamExt};
use tracing::Span;
use tracing_indicatif::span_ext::IndicatifSpanExt;
use tracing::Level;
@ -44,7 +46,7 @@ pub mod fs;
/// map and upload it to the [DirectoryService] through a lazily created [DirectoryPutter].
///
/// On success, returns the root node.
#[instrument(skip_all, ret(level = Level::TRACE), err)]
#[instrument(skip_all, fields(indicatif.pb_show=1), ret(level = Level::TRACE), err)]
pub async fn ingest_entries<DS, S, E>(
directory_service: DS,
mut entries: S,
@ -58,6 +60,8 @@ where
let mut directories: HashMap<PathBuf, Directory> = HashMap::default();
let mut maybe_directory_putter: Option<Box<dyn DirectoryPutter>> = None;
Span::current().pb_start();
let root_node = loop {
let mut entry = entries
.next()