maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Cache path info lookups in SQLite

Browse source 
This re-implements the binary cache database in C++, allowing it to be
used by other Store backends, in particular the S3 backend.
This commit is contained in:
Eelco Dolstra 2016-04-20 14:12:38 +02:00
parent e0204f8d46
commit 451ebf24ce
18 changed files with 380 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

3
src/libstore/binary-cache-store.cc
View file

@ -59,7 +59,7 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
narInfo->narSize = nar.size();
narInfo->narHash = hashString(htSHA256, nar);
if (info.narHash.type != htUnknown && info.narHash != narInfo->narHash)
if (info.narHash && info.narHash != narInfo->narHash)
throw Error(format("refusing to copy corrupted path ‘%1%’ to binary cache") % info.path);
/* Compress the NAR. */
@ -96,7 +96,6 @@ void BinaryCacheStore::addToCache(const ValidPathInfo & info,
{
auto state_(state.lock());
state_->pathInfoCache.upsert(narInfo->path, std::shared_ptr<NarInfo>(narInfo));
stats.pathInfoCacheSize = state_->pathInfoCache.size();
}
stats.narInfoWrite++;

2
src/libstore/derivations.cc
View file

@ -290,7 +290,7 @@ Hash hashDerivationModulo(Store & store, Derivation drv)
DerivationInputs inputs2;
for (auto & i : drv.inputDrvs) {
Hash h = drvHashes[i.first];
if (h.type == htUnknown) {
if (!h) {
assert(store.isValidPath(i.first));
Derivation drv2 = readDerivation(i.first);
h = hashDerivationModulo(store, drv2);

2
src/libstore/download.cc
View file

@ -225,7 +225,7 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
{
auto url = resolveUri(url_);
Path cacheDir = getEnv("XDG_CACHE_HOME", getEnv("HOME", "") + "/.cache") + "/nix/tarballs";
Path cacheDir = getCacheDir() + "/nix/tarballs";
createDirs(cacheDir);
string urlHash = printHash32(hashString(htSHA256, url));

11
src/libstore/http-binary-cache-store.cc
View file

@ -1,6 +1,7 @@
#include "binary-cache-store.hh"
#include "download.hh"
#include "globals.hh"
#include "nar-info-disk-cache.hh"
namespace nix {
@ -24,13 +25,23 @@ public:
{
if (cacheUri.back() == '/')
cacheUri.pop_back();
diskCache = getNarInfoDiskCache();
}
std::string getUri() override
{
return cacheUri;
}
void init() override
{
// FIXME: do this lazily?
if (!diskCache->cacheExists(cacheUri)) {
if (!fileExists("nix-cache-info"))
throw Error(format("‘%s’ does not appear to be a binary cache") % cacheUri);
diskCache->createCache(cacheUri);
}
}
protected:

2
src/libstore/local-store.cc
View file

@ -580,7 +580,6 @@ uint64_t LocalStore::addValidPath(State & state,
{
auto state_(Store::state.lock());
state_->pathInfoCache.upsert(info.path, std::make_shared<ValidPathInfo>(info));
stats.pathInfoCacheSize = state_->pathInfoCache.size();
}
return id;
@ -1069,7 +1068,6 @@ void LocalStore::invalidatePath(State & state, const Path & path)
{
auto state_(Store::state.lock());
state_->pathInfoCache.erase(path);
stats.pathInfoCacheSize = state_->pathInfoCache.size();
}
}

217
src/libstore/nar-info-disk-cache.cc Normal file
View file

@ -0,0 +1,217 @@
#include "nar-info-disk-cache.hh"
#include "sync.hh"
#include "sqlite.hh"
#include "globals.hh"
#include <sqlite3.h>
namespace nix {
static const char * schema = R"sql(
create table if not exists BinaryCaches (
id integer primary key autoincrement not null,
url text unique not null,
timestamp integer not null,
storeDir text not null,
wantMassQuery integer not null,
priority integer not null
);
create table if not exists NARs (
cache integer not null,
storePath text not null,
url text,
compression text,
fileHash text,
fileSize integer,
narHash text,
narSize integer,
refs text,
deriver text,
sigs text,
timestamp integer not null,
primary key (cache, storePath),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
create table if not exists NARExistence (
cache integer not null,
storePath text not null,
exist integer not null,
timestamp integer not null,
primary key (cache, storePath),
foreign key (cache) references BinaryCaches(id) on delete cascade
);
)sql";
class NarInfoDiskCacheImpl : public NarInfoDiskCache
{
public:
/* How long negative lookups are valid. */
const int ttlNegative = 3600;
struct State
{
SQLite db;
SQLiteStmt insertCache, queryCache, insertNAR, queryNAR, insertNARExistence, queryNARExistence;
std::map<std::string, int> caches;
};
Sync<State> _state;
NarInfoDiskCacheImpl()
{
auto state(_state.lock());
Path dbPath = getCacheDir() + "/nix/binary-cache-v3.sqlite";
createDirs(dirOf(dbPath));
if (sqlite3_open_v2(dbPath.c_str(), &state->db.db,
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
throw Error(format("cannot open store cache ‘%s’") % dbPath);
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(state->db, "setting timeout");
// We can always reproduce the cache.
if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "making database asynchronous");
if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "setting journal mode");
if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK)
throwSQLiteError(state->db, "initialising database schema");
state->insertCache.create(state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
state->queryCache.create(state->db,
"select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?");
state->insertNAR.create(state->db,
"insert or replace into NARs(cache, storePath, url, compression, fileHash, fileSize, narHash, "
"narSize, refs, deriver, sigs, timestamp) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
state->queryNAR.create(state->db,
"select * from NARs where cache = ? and storePath = ?");
state->insertNARExistence.create(state->db,
"insert or replace into NARExistence(cache, storePath, exist, timestamp) values (?, ?, ?, ?)");
state->queryNARExistence.create(state->db,
"select exist, timestamp from NARExistence where cache = ? and storePath = ?");
}
int uriToInt(State & state, const std::string & uri)
{
auto i = state.caches.find(uri);
if (i == state.caches.end()) abort();
return i->second;
}
void createCache(const std::string & uri) override
{
auto state(_state.lock());
// FIXME: race
state->insertCache.use()(uri)(time(0))(settings.nixStore)(1)(0).exec();
assert(sqlite3_changes(state->db) == 1);
state->caches[uri] = sqlite3_last_insert_rowid(state->db);
}
bool cacheExists(const std::string & uri) override
{
auto state(_state.lock());
auto i = state->caches.find(uri);
if (i != state->caches.end()) return true;
auto queryCache(state->queryCache.use()(uri));
if (queryCache.next()) {
state->caches[uri] = queryCache.getInt(0);
return true;
}
return false;
}
std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const Path & storePath) override
{
auto state(_state.lock());
auto queryNAR(state->queryNAR.use()
(uriToInt(*state, uri))
(baseNameOf(storePath)));
if (!queryNAR.next())
// FIXME: check NARExistence
return {oUnknown, 0};
auto narInfo = make_ref<NarInfo>();
// FIXME: implement TTL.
narInfo->path = storePath;
narInfo->url = queryNAR.getStr(2);
narInfo->compression = queryNAR.getStr(3);
if (!queryNAR.isNull(4))
narInfo->fileHash = parseHash(queryNAR.getStr(4));
narInfo->fileSize = queryNAR.getInt(5);
narInfo->narHash = parseHash(queryNAR.getStr(6));
narInfo->narSize = queryNAR.getInt(7);
for (auto & r : tokenizeString<Strings>(queryNAR.getStr(8), " "))
narInfo->references.insert(settings.nixStore + "/" + r);
if (!queryNAR.isNull(9))
narInfo->deriver = settings.nixStore + "/" + queryNAR.getStr(9);
for (auto & sig : tokenizeString<Strings>(queryNAR.getStr(10), " "))
narInfo->sigs.insert(sig);
return {oValid, narInfo};
}
void upsertNarInfo(
const std::string & uri, std::shared_ptr<ValidPathInfo> info) override
{
auto state(_state.lock());
if (info) {
auto narInfo = std::dynamic_pointer_cast<NarInfo>(info);
state->insertNAR.use()
(uriToInt(*state, uri))
(baseNameOf(info->path))
(narInfo ? narInfo->url : "", narInfo != 0)
(narInfo ? narInfo->compression : "", narInfo != 0)
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash)
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
(info->narHash.to_string())
(info->narSize)
(concatStringsSep(" ", info->shortRefs()))
(info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "")
(concatStringsSep(" ", info->sigs))
(time(0)).exec();
} else {
// not implemented
abort();
}
}
};
ref<NarInfoDiskCache> getNarInfoDiskCache()
{
static Sync<std::shared_ptr<NarInfoDiskCache>> cache;
auto cache_(cache.lock());
if (!*cache_) *cache_ = std::make_shared<NarInfoDiskCacheImpl>();
return ref<NarInfoDiskCache>(*cache_);
}
}

28
src/libstore/nar-info-disk-cache.hh Normal file
View file

@ -0,0 +1,28 @@
#pragma once
#include "ref.hh"
#include "nar-info.hh"
namespace nix {
class NarInfoDiskCache
{
public:
typedef enum { oValid, oInvalid, oUnknown } Outcome;
virtual void createCache(const std::string & uri) = 0;
virtual bool cacheExists(const std::string & uri) = 0;
virtual std::pair<Outcome, std::shared_ptr<NarInfo>> lookupNarInfo(
const std::string & uri, const Path & storePath) = 0;
virtual void upsertNarInfo(
const std::string & uri, std::shared_ptr<ValidPathInfo> narInfo) = 0;
};
/* Return a singleton cache object that can be used concurrently by
multiple threads. */
ref<NarInfoDiskCache> getNarInfoDiskCache();
}

20
src/libstore/nar-info.cc
View file

@ -5,16 +5,16 @@ namespace nix {
NarInfo::NarInfo(const std::string & s, const std::string & whence)
{
auto corrupt = [&]() {
auto corrupt = [&]() [[noreturn]] {
throw Error("NAR info file ‘%1%’ is corrupt");
};
auto parseHashField = [&](const string & s) {
string::size_type colon = s.find(':');
if (colon == string::npos) corrupt();
HashType ht = parseHashType(string(s, 0, colon));
if (ht == htUnknown) corrupt();
return parseHash16or32(ht, string(s, colon + 1));
try {
return parseHash(s);
} catch (BadHash &) {
corrupt();
}
};
size_t pos = 0;
@ -103,12 +103,4 @@ std::string NarInfo::to_string() const
return res;
}
Strings NarInfo::shortRefs() const
{
Strings refs;
for (auto & r : references)
refs.push_back(baseNameOf(r));
return refs;
}
}

4
src/libstore/nar-info.hh
View file

@ -19,10 +19,6 @@ struct NarInfo : ValidPathInfo
NarInfo(const std::string & s, const std::string & whence);
std::string to_string() const;
private:
Strings shortRefs() const;
};
}

1
src/libstore/remote-store.cc
View file

@ -489,7 +489,6 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
{
auto state_(Store::state.lock());
state_->pathInfoCache.clear();
stats.pathInfoCacheSize = 0;
}
}

5
src/libstore/sqlite.cc
View file

@ -139,6 +139,11 @@ int64_t SQLiteStmt::Use::getInt(int col)
return sqlite3_column_int64(stmt, col);
}
bool SQLiteStmt::Use::isNull(int col)
{
return sqlite3_column_type(stmt, col) == SQLITE_NULL;
}
SQLiteTxn::SQLiteTxn(sqlite3 * db)
{
this->db = db;

1
src/libstore/sqlite.hh
View file

@ -58,6 +58,7 @@ struct SQLiteStmt
std::string getStr(int col);
int64_t getInt(int col);
bool isNull(int col);
};
Use use()

50
src/libstore/store-api.cc
View file

@ -2,6 +2,7 @@
#include "globals.hh"
#include "store-api.hh"
#include "util.hh"
#include "nar-info-disk-cache.hh"
namespace nix {
@ -225,6 +226,12 @@ Path computeStorePathForText(const string & name, const string & s,
}
std::string Store::getUri()
{
return "";
}
bool Store::isValidPath(const Path & storePath)
{
{
@ -236,7 +243,19 @@ bool Store::isValidPath(const Path & storePath)
}
}
if (diskCache) {
auto res = diskCache->lookupNarInfo(getUri(), storePath);
if (res.first != NarInfoDiskCache::oUnknown) {
auto state_(state.lock());
state_->pathInfoCache.upsert(storePath,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
return res.first == NarInfoDiskCache::oValid;
}
}
return isValidPathUncached(storePath);
// FIXME: insert result into NARExistence table of diskCache.
}
@ -253,12 +272,26 @@ ref<const ValidPathInfo> Store::queryPathInfo(const Path & storePath)
}
}
if (diskCache) {
auto res = diskCache->lookupNarInfo(getUri(), storePath);
if (res.first != NarInfoDiskCache::oUnknown) {
auto state_(state.lock());
state_->pathInfoCache.upsert(storePath,
res.first == NarInfoDiskCache::oInvalid ? 0 : res.second);
if (res.first == NarInfoDiskCache::oInvalid)
throw InvalidPath(format("path ‘%s’ is not valid") % storePath);
return ref<ValidPathInfo>(res.second);
}
}
auto info = queryPathInfoUncached(storePath);
if (diskCache && info)
diskCache->upsertNarInfo(getUri(), info);
{
auto state_(state.lock());
state_->pathInfoCache.upsert(storePath, info);
stats.pathInfoCacheSize = state_->pathInfoCache.size();
}
if (!info) {
@ -303,6 +336,10 @@ string Store::makeValidityRegistration(const PathSet & paths,
const Store::Stats & Store::getStats()
{
{
auto state_(state.lock());
stats.pathInfoCacheSize = state_->pathInfoCache.size();
}
return stats;
}
@ -356,7 +393,7 @@ void Store::exportPaths(const Paths & paths,
std::string ValidPathInfo::fingerprint() const
{
if (narSize == 0 || narHash.type == htUnknown)
if (narSize == 0 || !narHash)
throw Error(format("cannot calculate fingerprint of path ‘%s’ because its size/hash is not known")
% path);
return
@ -389,6 +426,15 @@ bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::str
}
Strings ValidPathInfo::shortRefs() const
{
Strings refs;
for (auto & r : references)
refs.push_back(baseNameOf(r));
return refs;
}
}

7
src/libstore/store-api.hh
View file

@ -134,6 +134,8 @@ struct ValidPathInfo
/* Verify a single signature. */
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
Strings shortRefs() const;
virtual ~ValidPathInfo() { }
};
@ -170,6 +172,7 @@ struct BuildResult
struct BasicDerivation;
struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store : public std::enable_shared_from_this<Store>
@ -183,10 +186,14 @@ protected:
Sync<State> state;
std::shared_ptr<NarInfoDiskCache> diskCache;
public:
virtual ~Store() { }
virtual std::string getUri();
/* Check whether a path is valid. */
bool isValidPath(const Path & path);

32
src/libutil/hash.cc
View file

@ -33,7 +33,7 @@ Hash::Hash(HashType type)
else if (type == htSHA1) hashSize = sha1HashSize;
else if (type == htSHA256) hashSize = sha256HashSize;
else if (type == htSHA512) hashSize = sha512HashSize;
else throw Error("unknown hash type");
else abort();
assert(hashSize <= maxHashSize);
memset(hash, 0, maxHashSize);
}
@ -64,6 +64,12 @@ bool Hash::operator < (const Hash & h) const
}
std::string Hash::to_string(bool base32) const
{
return printHashType(type) + ":" + (base32 ? printHash32(*this) : printHash(*this));
}
const string base16Chars = "0123456789abcdef";
@ -78,15 +84,28 @@ string printHash(const Hash & hash)
}
Hash parseHash(const string & s)
{
string::size_type colon = s.find(':');
if (colon == string::npos)
throw BadHash(format("invalid hash ‘%s’") % s);
string hts = string(s, 0, colon);
HashType ht = parseHashType(hts);
if (ht == htUnknown)
throw BadHash(format("unknown hash type ‘%s’") % hts);
return parseHash16or32(ht, string(s, colon + 1));
}
Hash parseHash(HashType ht, const string & s)
{
Hash hash(ht);
if (s.length() != hash.hashSize * 2)
throw Error(format("invalid hash ‘%1%’") % s);
throw BadHash(format("invalid hash ‘%1%’") % s);
for (unsigned int i = 0; i < hash.hashSize; i++) {
string s2(s, i * 2, 2);
if (!isxdigit(s2[0]) || !isxdigit(s2[1]))
throw Error(format("invalid hash ‘%1%’") % s);
throw BadHash(format("invalid hash ‘%1%’") % s);
std::istringstream str(s2);
int n;
str >> std::hex >> n;
@ -103,6 +122,7 @@ const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz";
string printHash32(const Hash & hash)
{
size_t len = hash.base32Len();
assert(len);
string s;
s.reserve(len);
@ -139,7 +159,7 @@ Hash parseHash32(HashType ht, const string & s)
for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */
if (base32Chars[digit] == c) break;
if (digit >= 32)
throw Error(format("invalid base-32 hash ‘%1%’") % s);
throw BadHash(format("invalid base-32 hash ‘%1%’") % s);
unsigned int b = n * 5;
unsigned int i = b / 8;
unsigned int j = b % 8;
@ -161,7 +181,7 @@ Hash parseHash16or32(HashType ht, const string & s)
/* base-32 representation */
hash = parseHash32(ht, s);
else
throw Error(format("hash ‘%1%’ has wrong length for hash type ‘%2%’")
throw BadHash(format("hash ‘%1%’ has wrong length for hash type ‘%2%’")
% s % printHashType(ht));
return hash;
}
@ -322,7 +342,7 @@ string printHashType(HashType ht)
else if (ht == htSHA1) return "sha1";
else if (ht == htSHA256) return "sha256";
else if (ht == htSHA512) return "sha512";
else throw Error("cannot print unknown hash type");
else abort();
}

12
src/libutil/hash.hh
View file

@ -7,6 +7,9 @@
namespace nix {
MakeError(BadHash, Error);
enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 };
@ -26,12 +29,15 @@ struct Hash
HashType type;
/* Create an unusable hash object. */
/* Create an unset hash object. */
Hash();
/* Create a zero-filled hash object. */
Hash(HashType type);
/* Check whether a hash is set. */
operator bool () const { return type != htUnknown; }
/* Check whether two hash are equal. */
bool operator == (const Hash & h2) const;
@ -52,12 +58,16 @@ struct Hash
{
return (hashSize * 8 - 1) / 5 + 1;
}
std::string to_string(bool base32 = true) const;
};
/* Convert a hash to a hexadecimal representation. */
string printHash(const Hash & hash);
Hash parseHash(const string & s);
/* Parse a hexadecimal representation of a hash code. */
Hash parseHash(HashType ht, const string & s);

12
src/libutil/util.cc
View file

@ -403,6 +403,18 @@ Path createTempDir(const Path & tmpRoot, const Path & prefix,
}
Path getCacheDir()
{
Path cacheDir = getEnv("XDG_CACHE_HOME");
if (cacheDir.empty()) {
Path homeDir = getEnv("HOME");
if (homeDir.empty()) throw Error("$XDG_CACHE_HOME and $HOME are not set");
cacheDir = homeDir + "/.cache";
}
return cacheDir;
}
Paths createDirs(const Path & path)
{
Paths created;

3
src/libutil/util.hh
View file

@ -102,6 +102,9 @@ void deletePath(const Path & path, unsigned long long & bytesFreed);
Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix",
bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755);
/* Return the path to $XDG_CACHE_HOME/.cache. */
Path getCacheDir();
/* Create a directory and all its parents, if necessary. Returns the
list of created directories, in order of creation. */
Paths createDirs(const Path & path);