maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

S3BinaryCacheStore: Support compression of narinfo and log files

Browse source 
You can now set the store parameter "text-compression=br" to compress
textual files in the binary cache (i.e. narinfo and logs) using
Brotli. This sets the Content-Encoding header; the extension of
compressed files is unchanged.

You can separately specify the compression of log files using
"log-compression=br". This is useful when you don't want to compress
narinfo files for backward compatibility.
This commit is contained in:
Eelco Dolstra 2017-03-14 15:03:53 +01:00
parent 2691498b5c
commit 8b1d65bebe
No known key found for this signature in database
GPG key ID: 8170B4726D7198DE
5 changed files with 71 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

28
src/libstore/s3-binary-cache-store.cc
View file

@ -5,6 +5,8 @@
#include "nar-info.hh"
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
#include "download.hh"
#include <aws/core/Aws.h>
#include <aws/core/client/ClientConfiguration.h>
@ -104,8 +106,10 @@ S3Helper::DownloadResult S3Helper::getObject(
auto result = checkAws(fmt("AWS error fetching ‘%s’", key),
client->GetObject(request));
res.data = std::make_shared<std::string>(
dynamic_cast<std::stringstream &>(result.GetBody()).str());
res.data = decodeContent(
result.GetContentEncoding(),
make_ref<std::string>(
dynamic_cast<std::stringstream &>(result.GetBody()).str()));
} catch (S3Error & e) {
if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
@ -137,11 +141,15 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
S3Helper s3Helper;
std::string textCompression, logCompression;
S3BinaryCacheStoreImpl(
const Params & params, const std::string & bucketName)
: S3BinaryCacheStore(params)
, bucketName(bucketName)
, s3Helper(get(params, "aws-region", Aws::Region::US_EAST_1))
, textCompression(get(params, "text-compression", "gzip"))
, logCompression(get(params, "log-compression", textCompression))
{
diskCache = getNarInfoDiskCache();
}
@ -220,13 +228,17 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
return true;
}
void upsertFile(const std::string & path, const std::string & data) override
void uploadFile(const std::string & path, const std::string & data,
const std::string & contentEncoding)
{
auto request =
Aws::S3::Model::PutObjectRequest()
.WithBucket(bucketName)
.WithKey(path);
if (contentEncoding != "")
request.SetContentEncoding(contentEncoding);
auto stream = std::make_shared<istringstream_nocopy>(data);
request.SetBody(stream);
@ -249,6 +261,16 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore
stats.putTimeMs += duration;
}
void upsertFile(const std::string & path, const std::string & data) override
{
if (path.find(".narinfo") != std::string::npos)
uploadFile(path, *compress(textCompression, data), textCompression);
else if (path.find("/log") != std::string::npos)
uploadFile(path, *compress(logCompression, data), logCompression);
else
uploadFile(path, data, "");
}
void getFile(const std::string & path,
std::function<void(std::shared_ptr<std::string>)> success,
std::function<void(std::exception_ptr exc)> failure) override