Put a time-to-live on negative binary cache lookups
Negative lookups are purged from the DB after a day, at most once per day. However, for non-"have" lookups (e.g. all except "nix-env -qas"), negative lookups are ignored after one hour. This is to ensure that you don't have to wait a day for an operation like "nix-env -i" to start using new binaries in the cache. Should probably make this configurable.
This commit is contained in:
		
							parent
							
								
									e03a8a1c92
								
							
						
					
					
						commit
						47ae3ce2ca
					
				
					 1 changed files with 30 additions and 3 deletions
				
			
		| 
						 | 
				
			
			@ -19,10 +19,14 @@ my $gotCaches = 0;
 | 
			
		|||
my $maxParallelRequests = int($Nix::Config::config{"binary-caches-parallel-connections"} // 150);
 | 
			
		||||
$maxParallelRequests = 1 if $maxParallelRequests < 1;
 | 
			
		||||
 | 
			
		||||
my $ttlNegative = 24 * 3600; # when to purge negative lookups from the database
 | 
			
		||||
my $ttlNegativeUse = 3600; # how long negative lookups are valid for non-"have" lookups
 | 
			
		||||
my $didExpiration = 0;
 | 
			
		||||
 | 
			
		||||
my $debug = ($ENV{"NIX_DEBUG_SUBST"} // "") eq 1;
 | 
			
		||||
open(STDERR, ">>/dev/tty") if $debug;
 | 
			
		||||
 | 
			
		||||
my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence);
 | 
			
		||||
my ($dbh, $queryCache, $insertNAR, $queryNAR, $insertNARExistence, $queryNARExistence, $expireNARExistence);
 | 
			
		||||
 | 
			
		||||
my $curlm = WWW::Curl::Multi->new;
 | 
			
		||||
my $activeRequests = 0;
 | 
			
		||||
| 
						 | 
				
			
			@ -149,6 +153,8 @@ EOF
 | 
			
		|||
        );
 | 
			
		||||
EOF
 | 
			
		||||
 | 
			
		||||
    $dbh->do("create index if not exists NARExistenceByExistTimestamp on NARExistence (exist, timestamp)");
 | 
			
		||||
 | 
			
		||||
    $queryCache = $dbh->prepare("select id, storeDir, wantMassQuery from BinaryCaches where url = ?") or die;
 | 
			
		||||
 | 
			
		||||
    $insertNAR = $dbh->prepare(
 | 
			
		||||
| 
						 | 
				
			
			@ -160,7 +166,9 @@ EOF
 | 
			
		|||
    $insertNARExistence = $dbh->prepare(
 | 
			
		||||
        "insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)") or die;
 | 
			
		||||
 | 
			
		||||
    $queryNARExistence = $dbh->prepare("select exist from NARExistence where cache = ? and storePath = ?") or die;
 | 
			
		||||
    $queryNARExistence = $dbh->prepare("select exist, timestamp from NARExistence where cache = ? and storePath = ?") or die;
 | 
			
		||||
 | 
			
		||||
    $expireNARExistence = $dbh->prepare("delete from NARExistence where exist = ? and timestamp < ?") or die;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -238,6 +246,8 @@ sub getAvailableCaches {
 | 
			
		|||
        next if $storeDir ne $Nix::Config::storeDir;
 | 
			
		||||
        push @caches, { id => $id, url => $url, wantMassQuery => $wantMassQuery };
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
    expireNegative();
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -324,7 +334,7 @@ sub negativeHit {
 | 
			
		|||
    my ($storePath, $cache) = @_;
 | 
			
		||||
    $queryNARExistence->execute($cache->{id}, basename($storePath));
 | 
			
		||||
    my $res = $queryNARExistence->fetchrow_hashref();
 | 
			
		||||
    return defined $res && $res->{exist} == 0;
 | 
			
		||||
    return defined $res && $res->{exist} == 0 && time() - $res->{timestamp} < $ttlNegativeUse;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -337,6 +347,21 @@ sub positiveHit {
 | 
			
		|||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
sub expireNegative {
 | 
			
		||||
    return if $didExpiration;
 | 
			
		||||
    $didExpiration = 1;
 | 
			
		||||
    my $time = time();
 | 
			
		||||
    # Round up to the next multiple of the TTL to ensure that we do
 | 
			
		||||
    # expiration only once per time interval.  E.g. if $ttlNegative ==
 | 
			
		||||
    # 3600, we expire entries at most once per hour.  This is
 | 
			
		||||
    # presumably faster than expiring a few entries per request (and
 | 
			
		||||
    # thus doing a transaction).
 | 
			
		||||
    my $limit = (int($time / $ttlNegative) - 1) * $ttlNegative;
 | 
			
		||||
    $expireNARExistence->execute($limit, 0);
 | 
			
		||||
    print STDERR "expired ", $expireNARExistence->rows, " negative entries\n" if $debug;
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
sub printInfo {
 | 
			
		||||
    my ($storePath, $info) = @_;
 | 
			
		||||
    print "$storePath\n";
 | 
			
		||||
| 
						 | 
				
			
			@ -512,11 +537,13 @@ if ($ARGV[0] eq "--query") {
 | 
			
		|||
        my ($cmd, @args) = split " ", $_;
 | 
			
		||||
 | 
			
		||||
        if ($cmd eq "have") {
 | 
			
		||||
            print STDERR "checking binary caches for existence of @args\n" if $debug;
 | 
			
		||||
            printSubstitutablePaths(@args);
 | 
			
		||||
            print "\n";
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        elsif ($cmd eq "info") {
 | 
			
		||||
            print STDERR "checking binary caches for info on @args\n" if $debug;
 | 
			
		||||
            printInfoParallel(@args);
 | 
			
		||||
            print "\n";
 | 
			
		||||
        }
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue