maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(users/edef/weave/swizzle): use polars streaming

Browse source 
This vastly reduces the memory requirements, so we can run in ~40G RAM.

Change-Id: I4952a780df294bd852a8b4682ba2fd59b9bae675
Reviewed-on: https://cl.tvl.fyi/c/depot/+/12667
Reviewed-by: flokli <flokli@flokli.de>
Tested-by: BuildkiteCI
This commit is contained in:
edef 2024-10-17 13:26:01 +00:00
parent bdc2891053
commit 313899c291
5 changed files with 70 additions and 112 deletions
Download patch file Download diff file Expand all files Collapse all files

3
users/edef/weave/Cargo.toml
View file

@ -13,8 +13,7 @@ hashbrown = "0.14.3"
nix-compat = { version = "0.1.0", path = "../../../tvix/nix-compat" }
safer_owning_ref = "0.5.0"
rayon = "1.8.1"
tokio = { version = "1.36.0", features = ["full"] }
[dependencies.polars]
version = "0.36.2"
features = ["parquet"]
features = ["parquet", "lazy", "streaming"]