maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #1857 from dtzWill/fix/check-for-lzma-mt

Browse source 
configure.ac: check if lzma has MT support, fix deb build/etc.
This commit is contained in:
Eelco Dolstra 2018-02-19 13:06:14 +01:00 • committed by GitHub
commit 7fe5910bf8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 39 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

2
configure.ac
View file

@ -175,6 +175,8 @@ AC_SUBST(HAVE_SODIUM, [$have_sodium])
# Look for liblzma, a required dependency. # Look for liblzma, a required dependency.
PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"])
AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt],
[AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])])
# Look for libbrotli{enc,dec}, optional dependencies # Look for libbrotli{enc,dec}, optional dependencies

60
src/libutil/compression.cc
View file

@ -1,6 +1,7 @@
#include "compression.hh" #include "compression.hh"
#include "util.hh" #include "util.hh"
#include "finally.hh" #include "finally.hh"
#include "logging.hh"
#include <lzma.h> #include <lzma.h>
#include <bzlib.h> #include <bzlib.h>
@ -189,28 +190,9 @@ struct XzSink : CompressionSink
lzma_stream strm = LZMA_STREAM_INIT; lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false; bool finished = false;
XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink) template <typename F>
{ XzSink(Sink & nextSink, F&& initEncoder) : nextSink(nextSink) {
lzma_ret ret; lzma_ret ret = initEncoder();
if (parallel) {
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0)
mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
ret = lzma_stream_encoder_mt(
&strm, &mt_options);
} else
ret = lzma_easy_encoder(
&strm, 6, LZMA_CHECK_CRC64);
if (ret != LZMA_OK) if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder"); throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter? // FIXME: apply the x86 BCJ filter?
@ -218,6 +200,9 @@ struct XzSink : CompressionSink
strm.next_out = outbuf; strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf); strm.avail_out = sizeof(outbuf);
} }
XzSink(Sink & nextSink) : XzSink(nextSink, [this]() {
return lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64);
}) {}
~XzSink() ~XzSink()
{ {
@ -271,6 +256,27 @@ struct XzSink : CompressionSink
} }
}; };
#ifdef HAVE_LZMA_MT
struct ParallelXzSink : public XzSink
{
ParallelXzSink(Sink &nextSink) : XzSink(nextSink, [this]() {
lzma_mt mt_options = {};
mt_options.flags = 0;
mt_options.timeout = 300; // Using the same setting as the xz cmd line
mt_options.preset = LZMA_PRESET_DEFAULT;
mt_options.filters = NULL;
mt_options.check = LZMA_CHECK_CRC64;
mt_options.threads = lzma_cputhreads();
mt_options.block_size = 0;
if (mt_options.threads == 0)
mt_options.threads = 1;
// FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
// number of threads.
return lzma_stream_encoder_mt(&strm, &mt_options);
}) {}
};
#endif
struct BzipSink : CompressionSink struct BzipSink : CompressionSink
{ {
Sink & nextSink; Sink & nextSink;
@ -469,10 +475,18 @@ struct BrotliSink : CompressionSink
ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{ {
if (parallel) {
#ifdef HAVE_LZMA_MT
if (method == "xz")
return make_ref<ParallelXzSink>(nextSink);
#endif
printMsg(lvlError, format("Warning: parallel compression requested but not supported for method '%1%', falling back to single-threaded compression") % method);
}
if (method == "none") if (method == "none")
return make_ref<NoneSink>(nextSink); return make_ref<NoneSink>(nextSink);
else if (method == "xz") else if (method == "xz")
return make_ref<XzSink>(nextSink, parallel); return make_ref<XzSink>(nextSink);
else if (method == "bzip2") else if (method == "bzip2")
return make_ref<BzipSink>(nextSink); return make_ref<BzipSink>(nextSink);
else if (method == "br") else if (method == "br")