* `optimise-store.pl' reduces disk space consumption by hard-linking
all identitical files in the Nix store to each other. (Previously it only computed the size that would be saved by doing so.)
This commit is contained in:
		
							parent
							
								
									07cec27848
								
							
						
					
					
						commit
						6dbed1bf38
					
				
					 1 changed files with 40 additions and 16 deletions
				
			
		|  | @ -1,61 +1,85 @@ | ||||||
| #! /usr/bin/perl -w | #! /usr/bin/perl -w | ||||||
| 
 | 
 | ||||||
| use strict; | use strict; | ||||||
|  | use File::Basename; | ||||||
| 
 | 
 | ||||||
| #{ my $ofh = select STDOUT; |  | ||||||
| #  $| = 1; |  | ||||||
| #  select $ofh; |  | ||||||
| #} |  | ||||||
| 
 | 
 | ||||||
| #my @paths = ("/nix/store/d49mc94xwwd7wf1xzfh4ch4cypn0ajjr-glibc-2.3.6", "/nix/store/1mgfgy3ga4m9z60747s0yzxl0g6w5kxz-glibc-2.3.6"); |  | ||||||
| my @paths = ("/nix/store"); | my @paths = ("/nix/store"); | ||||||
| 
 | 
 | ||||||
|  | 
 | ||||||
|  | print "hashing...\n"; | ||||||
|  | 
 | ||||||
| my $hashList = "/tmp/nix-optimise-hash-list"; | my $hashList = "/tmp/nix-optimise-hash-list"; | ||||||
| 
 | 
 | ||||||
| system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0 | system("find @paths -type f -print0 | xargs -0 md5sum -- > $hashList") == 0 | ||||||
|     or die "cannot hash store files"; |     or die "cannot hash store files"; | ||||||
| 
 | 
 | ||||||
|  | 
 | ||||||
|  | print "sorting by hash...\n"; | ||||||
|  | 
 | ||||||
| system("sort $hashList > $hashList.sorted") == 0 | system("sort $hashList > $hashList.sorted") == 0 | ||||||
|     or die "cannot sort list"; |     or die "cannot sort list"; | ||||||
| 
 | 
 | ||||||
|  | 
 | ||||||
|  | sub atomicLink { | ||||||
|  |     my $target = shift; | ||||||
|  |     my $new = shift; | ||||||
|  |     my $tmpNew = "${new}_optimise.$$"; | ||||||
|  | 
 | ||||||
|  |     # Make the directory writable temporarily. | ||||||
|  |     my $dir = dirname $new; | ||||||
|  |     my @st = stat $dir or die; | ||||||
|  | 
 | ||||||
|  |     chmod ($st[2] | 0200, $dir) or die "cannot make `$dir' writable: $!"; | ||||||
|  |      | ||||||
|  |     link $target, $tmpNew or die "cannot create hard link `$tmpNew': $!"; | ||||||
|  | 
 | ||||||
|  |     rename $tmpNew, $new or die "cannot rename `$tmpNew' to `$new': $!"; | ||||||
|  | 
 | ||||||
|  |     chmod ($st[2], $dir) or die "cannot restore permission on `$dir': $!"; | ||||||
|  | } | ||||||
|  | 
 | ||||||
|  | 
 | ||||||
|  | print "hard-linking...\n"; | ||||||
|  | 
 | ||||||
| open LIST, "<$hashList.sorted" or die; | open LIST, "<$hashList.sorted" or die; | ||||||
| 
 | 
 | ||||||
| my $prevFile; | my $prevFile; | ||||||
| my $prevHash; | my $prevHash; | ||||||
|  | my $prevInode; | ||||||
| 
 | 
 | ||||||
| my $totalSpace = 0; | my $totalSpace = 0; | ||||||
| my $savedSpace = 0; | my $savedSpace = 0; | ||||||
| 
 | 
 | ||||||
| my $files = 0; |  | ||||||
| 
 |  | ||||||
| while (<LIST>) { | while (<LIST>) { | ||||||
|     /^([0-9a-f]*)\s+(.*)$/ or die; |     /^([0-9a-f]*)\s+(.*)$/ or die; | ||||||
|     my $curFile = $2; |     my $curFile = $2; | ||||||
|     my $curHash = $1; |     my $curHash = $1; | ||||||
| 
 | 
 | ||||||
|     my $fileSize = (stat $curFile)[7]; |     my @st = stat $curFile or die; | ||||||
|  |     next if ($st[2] & 0222) != 0; # skip writable files | ||||||
|  | 
 | ||||||
|  |     my $fileSize = $st[7]; | ||||||
|     $totalSpace += $fileSize; |     $totalSpace += $fileSize; | ||||||
| 
 | 
 | ||||||
|     if (defined $prevHash && $curHash eq $prevHash) { |     if (defined $prevHash && $curHash eq $prevHash) { | ||||||
|          |          | ||||||
|  |         if ($st[1] != $prevInode) { | ||||||
|             print "$curFile = $prevFile\n"; |             print "$curFile = $prevFile\n"; | ||||||
| 
 |             atomicLink $prevFile, $curFile; | ||||||
|             $savedSpace += $fileSize; |             $savedSpace += $fileSize; | ||||||
|  |         } | ||||||
|          |          | ||||||
|     } else { |     } else { | ||||||
|         $prevFile = $curFile; |         $prevFile = $curFile; | ||||||
|         $prevHash = $curHash; |         $prevHash = $curHash; | ||||||
|  |         $prevInode = $st[1]; | ||||||
|     } |     } | ||||||
| 
 |  | ||||||
| #    print "." if ($files++ % 100 == 0); |  | ||||||
| } | } | ||||||
| 
 | 
 | ||||||
| #print "\n"; |  | ||||||
| 
 |  | ||||||
| print "total space = $totalSpace\n"; | print "total space = $totalSpace\n"; | ||||||
| print "saved space = $savedSpace\n"; | print "saved space = $savedSpace\n"; | ||||||
| my $savings = ($savedSpace / $totalSpace) * 100.0; | my $savings = ($savedSpace / $totalSpace) * 100.0; | ||||||
| print "savings = $savings %\n"; | print "savings = $savings %\n"; | ||||||
| 
 | 
 | ||||||
| 
 |  | ||||||
| close LIST; | close LIST; | ||||||
|  |  | ||||||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue