Eelco Dolstra
GitHub
commit
6601e46567
13 changed files with 9 additions and 772 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -34,7 +34,6 @@ Makefile.config
|
||||||
|
|
||||||
# /scripts/
|
# /scripts/
|
||||||
/scripts/nix-profile.sh
|
/scripts/nix-profile.sh
|
||||||
/scripts/nix-push
|
|
||||||
/scripts/nix-switch
|
/scripts/nix-switch
|
||||||
/scripts/nix-collect-garbage
|
/scripts/nix-collect-garbage
|
||||||
/scripts/nix-prefetch-url
|
/scripts/nix-prefetch-url
|
||||||
|
|
|
||||||
|
|
@ -180,9 +180,7 @@ following files:</para>
|
||||||
sufficient rights to add binary caches. For instance, in a
|
sufficient rights to add binary caches. For instance, in a
|
||||||
multi-user Nix setup, the binary caches provided by the channels
|
multi-user Nix setup, the binary caches provided by the channels
|
||||||
of the root user are used automatically, but caches corresponding
|
of the root user are used automatically, but caches corresponding
|
||||||
to the channels of non-root users are ignored. Binary caches can
|
to the channels of non-root users are ignored.</para></listitem>
|
||||||
be created and maintained using
|
|
||||||
<command>nix-push</command>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
</varlistentry>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,449 +0,0 @@
|
||||||
<refentry xmlns="http://docbook.org/ns/docbook"
|
|
||||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
|
||||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
|
||||||
version="5.0"
|
|
||||||
xml:id="sec-nix-push">
|
|
||||||
|
|
||||||
<refmeta>
|
|
||||||
<refentrytitle>nix-push</refentrytitle>
|
|
||||||
<manvolnum>1</manvolnum>
|
|
||||||
<refmiscinfo class="source">Nix</refmiscinfo>
|
|
||||||
<refmiscinfo class="version"><xi:include href="../version.txt" parse="text"/></refmiscinfo>
|
|
||||||
</refmeta>
|
|
||||||
|
|
||||||
<refnamediv>
|
|
||||||
<refname>nix-push</refname>
|
|
||||||
<refpurpose>generate a binary cache</refpurpose>
|
|
||||||
</refnamediv>
|
|
||||||
|
|
||||||
<refsynopsisdiv>
|
|
||||||
<cmdsynopsis>
|
|
||||||
<command>nix-push</command>
|
|
||||||
<arg choice='plain'><option>--dest</option> <replaceable>dest-dir</replaceable></arg>
|
|
||||||
<arg><option>--bzip2</option></arg>
|
|
||||||
<arg><option>--none</option></arg>
|
|
||||||
<arg><option>--force</option></arg>
|
|
||||||
<arg><option>--link</option></arg>
|
|
||||||
<arg><option>--manifest</option></arg>
|
|
||||||
<arg><option>--manifest-path</option> <replaceable>filename</replaceable></arg>
|
|
||||||
<arg><option>--url-prefix</option> <replaceable>url</replaceable></arg>
|
|
||||||
<arg><option>--key-file</option> <replaceable>path</replaceable></arg>
|
|
||||||
<arg choice='plain' rep='repeat'><replaceable>paths</replaceable></arg>
|
|
||||||
</cmdsynopsis>
|
|
||||||
</refsynopsisdiv>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Description</title>
|
|
||||||
|
|
||||||
<para>The command <command>nix-push</command> produces a
|
|
||||||
<emphasis>binary cache</emphasis>, a directory containing compressed
|
|
||||||
Nix archives (NARs) plus some metadata of the closure of the specified
|
|
||||||
store paths. This directory can then be made available through a web
|
|
||||||
server to other Nix installations, allowing them to skip building from
|
|
||||||
source and instead download binaries from the cache
|
|
||||||
automatically.</para>
|
|
||||||
|
|
||||||
<para><command>nix-push</command> performs the following actions.
|
|
||||||
|
|
||||||
<orderedlist>
|
|
||||||
|
|
||||||
<listitem><para>Each path in <replaceable>paths</replaceable> is
|
|
||||||
built (using <link
|
|
||||||
linkend='rsec-nix-store-realise'><command>nix-store
|
|
||||||
--realise</command></link>).</para></listitem>
|
|
||||||
|
|
||||||
<listitem><para>All paths in the closure of
|
|
||||||
<replaceable>paths</replaceable> are determined (using
|
|
||||||
<command>nix-store --query --requisites
|
|
||||||
--include-outputs</command>). Note that since the
|
|
||||||
<option>--include-outputs</option> flag is used, if
|
|
||||||
<replaceable>paths</replaceable> includes a store derivation, you
|
|
||||||
get a combined source/binary distribution (e.g., source tarballs
|
|
||||||
will be included).</para></listitem>
|
|
||||||
|
|
||||||
<listitem><para>All store paths determined in the previous step are
|
|
||||||
packaged into a NAR (using <command>nix-store --dump</command>) and
|
|
||||||
compressed using <command>xz</command> or <command>bzip2</command>.
|
|
||||||
The resulting files have the extension <filename>.nar.xz</filename>
|
|
||||||
or <filename>.nar.bz2</filename>. Also for each store path, Nix
|
|
||||||
generates a file with extension <filename>.narinfo</filename>
|
|
||||||
containing metadata such as the references, cryptographic hash and
|
|
||||||
size of each path.</para></listitem>
|
|
||||||
|
|
||||||
<listitem><para>Optionally, a single <emphasis>manifest</emphasis>
|
|
||||||
file is created that contains the same metadata as the
|
|
||||||
<filename>.narinfo</filename> files. This is for compatibility with
|
|
||||||
Nix versions prior to 1.2.</para></listitem>
|
|
||||||
|
|
||||||
<listitem><para>A file named <option>nix-cache-info</option> is
|
|
||||||
placed in the destination directory. The existence of this file
|
|
||||||
marks the directory as a binary cache.</para></listitem>
|
|
||||||
|
|
||||||
</orderedlist>
|
|
||||||
|
|
||||||
</para>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Options</title>
|
|
||||||
|
|
||||||
<variablelist>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--dest</option> <replaceable>dest-dir</replaceable></term>
|
|
||||||
|
|
||||||
<listitem><para>Set the destination directory to
|
|
||||||
<replaceable>dir</replaceable>, which is created if it does not
|
|
||||||
exist. This flag is required.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--bzip2</option></term>
|
|
||||||
|
|
||||||
<listitem><para>Compress NARs using <command>bzip2</command>
|
|
||||||
instead of <command>xz</command>. The latter compresses about 30%
|
|
||||||
better on typical archives, decompresses about twice as fast, but
|
|
||||||
compresses a lot slower and is not supported by Nix prior to
|
|
||||||
version 1.2.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--none</option></term>
|
|
||||||
|
|
||||||
<listitem><para>Do not compress NARs.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--force</option></term>
|
|
||||||
|
|
||||||
<listitem><para>Overwrite <filename>.narinfo</filename> files if
|
|
||||||
they already exist.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--link</option></term>
|
|
||||||
|
|
||||||
<listitem><para>By default, NARs are generated in the Nix store
|
|
||||||
and then copied to <replaceable>dest-dir</replaceable>. If this
|
|
||||||
option is given, hard links are used instead. This only works if
|
|
||||||
<replaceable>dest-dir</replaceable> is on the same filesystem as
|
|
||||||
the Nix store.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--manifest</option></term>
|
|
||||||
|
|
||||||
<listitem><para>Force the generation of a manifest suitable for
|
|
||||||
use by old versions of Nix. The manifest is stored as
|
|
||||||
<filename><replaceable>dest-dir</replaceable>/MANIFEST</filename>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--manifest-path</option> <replaceable>filename</replaceable></term>
|
|
||||||
|
|
||||||
<listitem><para>Like <option>--manifest</option>, but store the
|
|
||||||
manifest in <replaceable>filename</replaceable>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--url-prefix</option> <replaceable>url</replaceable></term>
|
|
||||||
|
|
||||||
<listitem><para>Manifests are expected to contain the absolute
|
|
||||||
URLs of NARs. For generating these URLs, the prefix
|
|
||||||
<replaceable>url</replaceable> is used. It defaults to
|
|
||||||
<uri>file://<replaceable>dest-dir</replaceable></uri>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><option>--key-file</option> <replaceable>path</replaceable></term>
|
|
||||||
|
|
||||||
<listitem><para>Sign the binary cache using the secret key stored
|
|
||||||
in <replaceable>path</replaceable>. This secret key must have been
|
|
||||||
created using <command
|
|
||||||
linkend="rsec-nix-store-generate-binary-cache-key">nix-store
|
|
||||||
--generate-binary-cache-key</command>. Users of this binary cache
|
|
||||||
should add the corresponding public key to the option
|
|
||||||
<option>binary-cache-public-keys</option> in
|
|
||||||
<filename>nix.conf</filename>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
</variablelist>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Examples</title>
|
|
||||||
|
|
||||||
<para>To add the closure of Thunderbird to a binary cache:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-push --dest /tmp/cache $(nix-build -A thunderbird)
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
Assuming that <filename>/tmp/cache</filename> is exported by a web
|
|
||||||
server as <uri>http://example.org/cache</uri>, you can then use this
|
|
||||||
cache on another machine to speed up the installation of Thunderbird:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-build -A thunderbird --option binary-caches http://example.org/cache
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
Alternatively, you could add <literal>binary-caches =
|
|
||||||
http://example.org/cache</literal> to
|
|
||||||
<filename>nix.conf</filename>.</para>
|
|
||||||
|
|
||||||
<para>To also include build-time dependencies (such as source
|
|
||||||
tarballs):
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-push --dest /tmp/cache $(nix-instantiate -A thunderbird)
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>To generate a signed binary cache, you must first generate a key
|
|
||||||
pair, in this example called <literal>cache.example.org-1</literal>,
|
|
||||||
storing the secret key in <filename>./sk</filename> and the public key
|
|
||||||
in <filename>./pk</filename>:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-store --generate-binary-cache-key cache.example.org-1 sk pk
|
|
||||||
|
|
||||||
$ cat sk
|
|
||||||
cache.example.org-1:jcMRQYFo8pQKzTtimpQLIPeHkMYZjfhB24hGfwF+u9PuX8H8FO7q564+X3G/JDlqqIqGar3OXRRwS9N3Wh3vbw==
|
|
||||||
|
|
||||||
$ cat pk
|
|
||||||
cache.example.org-1:7l/B/BTu6ueuPl9xvyQ5aqiKhmq9zl0UcEvTd1od728=
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
You can then generate a binary cache signed with the secret key:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
$ nix-push --dest /tmp/cache --key-file ./sk $(type -p firefox)
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
Users who wish to verify the integrity of binaries downloaded from
|
|
||||||
your cache would add the following to their
|
|
||||||
<filename>nix.conf</filename>:
|
|
||||||
|
|
||||||
<programlisting>
|
|
||||||
binary-caches = http://cache.example.org
|
|
||||||
signed-binary-caches = *
|
|
||||||
binary-cache-public-keys = cache.example.org-1:7l/B/BTu6ueuPl9xvyQ5aqiKhmq9zl0UcEvTd1od728=
|
|
||||||
</programlisting>
|
|
||||||
|
|
||||||
Nix will then ignore any binary that has a missing, incorrect or
|
|
||||||
unrecognised signature.</para>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
<refsection><title>Binary cache format and operation</title>
|
|
||||||
|
|
||||||
<para>A binary cache with URL <replaceable>url</replaceable> only
|
|
||||||
denotes a valid binary cache if the file
|
|
||||||
<uri><replaceable>url</replaceable>/nix-cache-info</uri> exists. If
|
|
||||||
this file does not exist (or cannot be downloaded), the cache is
|
|
||||||
ignored. If it does exist, it must be a text file containing cache
|
|
||||||
properties. Here’s an example:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
StoreDir: /nix/store
|
|
||||||
WantMassQuery: 1
|
|
||||||
Priority: 10
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The properties that are currently supported are:
|
|
||||||
|
|
||||||
<variablelist>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>StoreDir</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The path of the Nix store to which this binary
|
|
||||||
cache applies. Binaries are not relocatable — a binary built for
|
|
||||||
<filename>/nix/store</filename> won’t generally work in
|
|
||||||
<filename>/home/alice/store</filename> — so to prevent binaries
|
|
||||||
from being used in a wrong store, a binary cache is only used if
|
|
||||||
its <literal>StoreDir</literal> matches the local Nix
|
|
||||||
configuration. The default is
|
|
||||||
<filename>/nix/store</filename>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>WantMassQuery</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>Query operations such as <command>nix-env
|
|
||||||
-qas</command> can cause thousands of cache queries, and thus
|
|
||||||
thousands of HTTP requests, to determine which packages are
|
|
||||||
available in binary form. While these requests are small, not
|
|
||||||
every server may appreciate a potential onslaught of queries. If
|
|
||||||
<literal>WantMassQuery</literal> is set to <literal>0</literal>
|
|
||||||
(default), “mass queries” such as <command>nix-env -qas</command>
|
|
||||||
will skip this cache. Thus a package may appear not to have a
|
|
||||||
binary substitute. However, the binary will still be used when
|
|
||||||
you actually install the package. If
|
|
||||||
<literal>WantMassQuery</literal> is set to <literal>1</literal>,
|
|
||||||
mass queries will use this cache.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>Priority</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>Each binary cache has a priority (defaulting to
|
|
||||||
50). Binary caches are checked for binaries in order of ascending
|
|
||||||
priority; thus a higher number denotes a lower priority. The
|
|
||||||
binary cache <uri>https://cache.nixos.org</uri> has priority
|
|
||||||
40.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
</variablelist>
|
|
||||||
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>Every time Nix needs to build some store path
|
|
||||||
<replaceable>p</replaceable>, it will check each configured binary
|
|
||||||
cache to see if it has a NAR file for <replaceable>p</replaceable>,
|
|
||||||
until it finds one. If no cache has a NAR, Nix will fall back to
|
|
||||||
building the path from source (if applicable). To see if a cache with
|
|
||||||
URL <replaceable>url</replaceable> has a binary for
|
|
||||||
<replaceable>p</replaceable>, Nix fetches
|
|
||||||
<replaceable>url/h</replaceable>, where <replaceable>h</replaceable>
|
|
||||||
is the hash part of <replaceable>p</replaceable>. Thus, if we have a
|
|
||||||
cache <uri>https://cache.nixos.org</uri> and we want to obtain the
|
|
||||||
store path
|
|
||||||
<screen>
|
|
||||||
/nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7
|
|
||||||
</screen>
|
|
||||||
then Nix will attempt to fetch
|
|
||||||
<screen>
|
|
||||||
https://cache.nixos.org/a8922c0h87iilxzzvwn2hmv8x210aqb9.narinfo
|
|
||||||
</screen>
|
|
||||||
(Commands such as <command>nix-env -qas</command> will issue an HTTP
|
|
||||||
HEAD request, since it only needs to know if the
|
|
||||||
<filename>.narinfo</filename> file exists.) The
|
|
||||||
<filename>.narinfo</filename> file is a simple text file that looks
|
|
||||||
like this:
|
|
||||||
|
|
||||||
<screen>
|
|
||||||
StorePath: /nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7
|
|
||||||
URL: nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2
|
|
||||||
Compression: bzip2
|
|
||||||
FileHash: sha256:0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70
|
|
||||||
FileSize: 24473768
|
|
||||||
NarHash: sha256:0s491y1h9hxj5ghiizlxk7ax6jwbha00zwn7lpyd5xg5bhf60vzg
|
|
||||||
NarSize: 109521136
|
|
||||||
References: 2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16 ...
|
|
||||||
Deriver: 7akyyc87ka32xwmqza9dvyg5pwx3j212-glibc-2.7.drv
|
|
||||||
Sig: cache.example.org-1:WepnSp2UT0odDpR3NRjPVhJBHmdBgSBSTbHpdh4SCz92nGXwFY82bkPEmISoC0hGqBXDXEmB6y3Ohgna3mMgDg==
|
|
||||||
</screen>
|
|
||||||
|
|
||||||
The fields are as follows:
|
|
||||||
|
|
||||||
<variablelist>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>StorePath</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The full store path, including the name part
|
|
||||||
(e.g., <literal>glibc-2.7</literal>). It must match the
|
|
||||||
requested store path.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>URL</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The URL of the NAR, relative to the binary cache
|
|
||||||
URL.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>Compression</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The compression method; either
|
|
||||||
<literal>xz</literal> or
|
|
||||||
<literal>bzip2</literal>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>FileHash</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The SHA-256 hash of the compressed
|
|
||||||
NAR.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>FileSize</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The size of the compressed NAR.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>NarHash</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The SHA-256 hash of the uncompressed NAR. This is
|
|
||||||
equal to the hash of the store path as returned by
|
|
||||||
<command>nix-store -q --hash
|
|
||||||
<replaceable>p</replaceable></command>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>NarSize</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The size of the uncompressed NAR.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>References</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The references of the store path, without the Nix
|
|
||||||
store prefix.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>Deriver</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The deriver of the store path, without the Nix
|
|
||||||
store prefix. This field is optional.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>System</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>The Nix platform type of this binary, if known.
|
|
||||||
This field is optional.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
<varlistentry><term><literal>Sig</literal></term>
|
|
||||||
|
|
||||||
<listitem><para>A signature of the the form
|
|
||||||
<literal><replaceable>key-name</replaceable>:<replaceable>sig</replaceable></literal>,
|
|
||||||
where <replaceable>key-name</replaceable> is the symbolic name of
|
|
||||||
the key pair used to sign and verify the cache
|
|
||||||
(e.g. <literal>cache.example.org-1</literal>), and
|
|
||||||
<replaceable>sig</replaceable> is the actual signature, computed
|
|
||||||
over the <varname>StorePath</varname>, <varname>NarHash</varname>,
|
|
||||||
<varname>NarSize</varname> and <varname>References</varname>
|
|
||||||
fields using the <link
|
|
||||||
xlink:href="http://ed25519.cr.yp.to/">Ed25519 public-key signature
|
|
||||||
system</link>.</para></listitem>
|
|
||||||
|
|
||||||
</varlistentry>
|
|
||||||
|
|
||||||
</variablelist>
|
|
||||||
|
|
||||||
</para>
|
|
||||||
|
|
||||||
<para>Thus, in our example, after recursively ensuring that the
|
|
||||||
references exist (e.g.,
|
|
||||||
<filename>/nix/store/2ma2k0ys8knh4an48n28vigcmc2z8773-linux-headers-2.6.23.16</filename>),
|
|
||||||
Nix will fetch <screen>
|
|
||||||
https://cache.nixos.org/nar/0zzjpdz46mdn74v09m053yczlz4am038g8r74iy8w43gx8801h70.nar.bz2
|
|
||||||
</screen> and decompress and unpack it to
|
|
||||||
<filename>/nix/store/a8922c0h87iilxzzvwn2hmv8x210aqb9-glibc-2.7</filename>.</para>
|
|
||||||
|
|
||||||
</refsection>
|
|
||||||
|
|
||||||
|
|
||||||
</refentry>
|
|
||||||
|
|
@ -1390,8 +1390,7 @@ parameters:
|
||||||
|
|
||||||
</orderedlist>
|
</orderedlist>
|
||||||
|
|
||||||
For an example, see the manual page for <command
|
</para>
|
||||||
linkend="sec-nix-push">nix-push</command>.</para>
|
|
||||||
|
|
||||||
</refsection>
|
</refsection>
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,5 @@ work with Nix.</para>
|
||||||
<xi:include href="nix-hash.xml" />
|
<xi:include href="nix-hash.xml" />
|
||||||
<xi:include href="nix-instantiate.xml" />
|
<xi:include href="nix-instantiate.xml" />
|
||||||
<xi:include href="nix-prefetch-url.xml" />
|
<xi:include href="nix-prefetch-url.xml" />
|
||||||
<xi:include href="nix-push.xml" />
|
|
||||||
|
|
||||||
</chapter>
|
</chapter>
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid
|
||||||
# Generate man pages.
|
# Generate man pages.
|
||||||
man-pages := $(foreach n, \
|
man-pages := $(foreach n, \
|
||||||
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
|
nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \
|
||||||
nix-collect-garbage.1 nix-push.1 \
|
nix-collect-garbage.1 \
|
||||||
nix-prefetch-url.1 nix-channel.1 \
|
nix-prefetch-url.1 nix-channel.1 \
|
||||||
nix-hash.1 nix-copy-closure.1 \
|
nix-hash.1 nix-copy-closure.1 \
|
||||||
nix.conf.5 nix-daemon.8, \
|
nix.conf.5 nix-daemon.8, \
|
||||||
|
|
|
||||||
|
|
@ -2,7 +2,6 @@ nix_bin_scripts := \
|
||||||
$(d)/nix-build \
|
$(d)/nix-build \
|
||||||
$(d)/nix-channel \
|
$(d)/nix-channel \
|
||||||
$(d)/nix-copy-closure \
|
$(d)/nix-copy-closure \
|
||||||
$(d)/nix-push
|
|
||||||
|
|
||||||
bin-scripts += $(nix_bin_scripts)
|
bin-scripts += $(nix_bin_scripts)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,296 +0,0 @@
|
||||||
#! @perl@ -w @perlFlags@
|
|
||||||
|
|
||||||
use utf8;
|
|
||||||
use strict;
|
|
||||||
use File::Basename;
|
|
||||||
use File::Path qw(mkpath);
|
|
||||||
use File::stat;
|
|
||||||
use File::Copy;
|
|
||||||
use MIME::Base64;
|
|
||||||
use Nix::Config;
|
|
||||||
use Nix::Store;
|
|
||||||
use Nix::Manifest;
|
|
||||||
use Nix::Utils;
|
|
||||||
|
|
||||||
binmode STDERR, ":encoding(utf8)";
|
|
||||||
|
|
||||||
my $tmpDir = mkTempDir("nix-push");
|
|
||||||
|
|
||||||
my $nixExpr = "$tmpDir/create-nars.nix";
|
|
||||||
|
|
||||||
|
|
||||||
# Parse the command line.
|
|
||||||
my $compressionType = "xz";
|
|
||||||
my $force = 0;
|
|
||||||
my $destDir;
|
|
||||||
my $writeManifest = 0;
|
|
||||||
my $manifestPath;
|
|
||||||
my $archivesURL;
|
|
||||||
my $link = 0;
|
|
||||||
my $secretKeyFile;
|
|
||||||
my @roots;
|
|
||||||
my @buildArgs;
|
|
||||||
|
|
||||||
for (my $n = 0; $n < scalar @ARGV; $n++) {
|
|
||||||
my $arg = $ARGV[$n];
|
|
||||||
|
|
||||||
if ($arg eq "--help") {
|
|
||||||
exec "man nix-push" or die;
|
|
||||||
} elsif ($arg eq "--bzip2") {
|
|
||||||
$compressionType = "bzip2";
|
|
||||||
} elsif ($arg eq "--none") {
|
|
||||||
$compressionType = "none";
|
|
||||||
} elsif ($arg eq "--force") {
|
|
||||||
$force = 1;
|
|
||||||
} elsif ($arg eq "--dest") {
|
|
||||||
$n++;
|
|
||||||
die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
|
|
||||||
$destDir = $ARGV[$n];
|
|
||||||
mkpath($destDir, 0, 0755);
|
|
||||||
} elsif ($arg eq "--manifest") {
|
|
||||||
$writeManifest = 1;
|
|
||||||
} elsif ($arg eq "--manifest-path") {
|
|
||||||
$n++;
|
|
||||||
die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
|
|
||||||
$manifestPath = $ARGV[$n];
|
|
||||||
$writeManifest = 1;
|
|
||||||
mkpath(dirname($manifestPath), 0, 0755);
|
|
||||||
} elsif ($arg eq "--url-prefix") {
|
|
||||||
$n++;
|
|
||||||
die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
|
|
||||||
$archivesURL = $ARGV[$n];
|
|
||||||
} elsif ($arg eq "--link") {
|
|
||||||
$link = 1;
|
|
||||||
} elsif ($arg eq "--key-file") {
|
|
||||||
$n++;
|
|
||||||
die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
|
|
||||||
$secretKeyFile = $ARGV[$n];
|
|
||||||
} elsif ($arg eq "--max-jobs" || $arg eq "-j") {
|
|
||||||
$n++;
|
|
||||||
die "$0: ‘$arg’ requires an argument\n" unless $n < scalar @ARGV;
|
|
||||||
push @buildArgs, ($arg, $ARGV[$n]);
|
|
||||||
} elsif (substr($arg, 0, 1) eq "-") {
|
|
||||||
die "$0: unknown flag ‘$arg’\n";
|
|
||||||
} else {
|
|
||||||
push @roots, $arg;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
die "$0: please specify a destination directory\n" if !defined $destDir;
|
|
||||||
|
|
||||||
$archivesURL = "file://$destDir" unless defined $archivesURL;
|
|
||||||
|
|
||||||
|
|
||||||
# From the given store paths, determine the set of requisite store
|
|
||||||
# paths, i.e, the paths required to realise them.
|
|
||||||
my %storePaths;
|
|
||||||
|
|
||||||
foreach my $path (@roots) {
|
|
||||||
# Get all paths referenced by the normalisation of the given
|
|
||||||
# Nix expression.
|
|
||||||
my $pid = open(READ,
|
|
||||||
"$Nix::Config::binDir/nix-store --query --requisites --force-realise " .
|
|
||||||
"--include-outputs '$path'|") or die;
|
|
||||||
|
|
||||||
while (<READ>) {
|
|
||||||
chomp;
|
|
||||||
die "bad: $_" unless /^\//;
|
|
||||||
$storePaths{$_} = "";
|
|
||||||
}
|
|
||||||
|
|
||||||
close READ or die "nix-store failed: $?";
|
|
||||||
}
|
|
||||||
|
|
||||||
my @storePaths = keys %storePaths;
|
|
||||||
|
|
||||||
|
|
||||||
# Don't create archives for files that are already in the binary cache.
|
|
||||||
my @storePaths2;
|
|
||||||
my %narFiles;
|
|
||||||
foreach my $storePath (@storePaths) {
|
|
||||||
my $pathHash = substr(basename($storePath), 0, 32);
|
|
||||||
my $narInfoFile = "$destDir/$pathHash.narinfo";
|
|
||||||
if (!$force && -e $narInfoFile) {
|
|
||||||
my $narInfo = parseNARInfo($storePath, readFile($narInfoFile), 0, $narInfoFile) or die "cannot read ‘$narInfoFile’\n";
|
|
||||||
my $narFile = "$destDir/$narInfo->{url}";
|
|
||||||
if (-e $narFile) {
|
|
||||||
print STDERR "skipping existing $storePath\n";
|
|
||||||
# Add the NAR info to $narFiles if we're writing a
|
|
||||||
# manifest.
|
|
||||||
$narFiles{$storePath} = [
|
|
||||||
{ url => ("$archivesURL/" . basename $narInfo->{url})
|
|
||||||
, hash => $narInfo->{fileHash}
|
|
||||||
, size => $narInfo->{fileSize}
|
|
||||||
, compressionType => $narInfo->{compression}
|
|
||||||
, narHash => $narInfo->{narHash}
|
|
||||||
, narSize => $narInfo->{narSize}
|
|
||||||
, references => join(" ", map { "$Nix::Config::storeDir/$_" } @{$narInfo->{refs}})
|
|
||||||
, deriver => $narInfo->{deriver} ? "$Nix::Config::storeDir/$narInfo->{deriver}" : undef
|
|
||||||
}
|
|
||||||
] if $writeManifest;
|
|
||||||
next;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
push @storePaths2, $storePath;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Create a list of Nix derivations that turn each path into a Nix
|
|
||||||
# archive.
|
|
||||||
open NIX, ">$nixExpr";
|
|
||||||
print NIX "[";
|
|
||||||
|
|
||||||
foreach my $storePath (@storePaths2) {
|
|
||||||
die unless ($storePath =~ /\/[0-9a-z]{32}[^\"\\\$]*$/);
|
|
||||||
|
|
||||||
# Construct a Nix expression that creates a Nix archive.
|
|
||||||
my $nixexpr =
|
|
||||||
"(import <nix/nar.nix> " .
|
|
||||||
"{ storePath = builtins.storePath \"$storePath\"; hashAlgo = \"sha256\"; compressionType = \"$compressionType\"; }) ";
|
|
||||||
|
|
||||||
print NIX $nixexpr;
|
|
||||||
}
|
|
||||||
|
|
||||||
print NIX "]";
|
|
||||||
close NIX;
|
|
||||||
|
|
||||||
|
|
||||||
# Build the Nix expression.
|
|
||||||
print STDERR "building compressed archives...\n";
|
|
||||||
my @narPaths;
|
|
||||||
my $pid = open(READ, "-|", "$Nix::Config::binDir/nix-build", $nixExpr, "-o", "$tmpDir/result", @buildArgs)
|
|
||||||
or die "cannot run nix-build";
|
|
||||||
while (<READ>) {
|
|
||||||
chomp;
|
|
||||||
die unless /^\//;
|
|
||||||
push @narPaths, $_;
|
|
||||||
}
|
|
||||||
close READ or die "nix-build failed: $?";
|
|
||||||
|
|
||||||
|
|
||||||
# Write the cache info file.
|
|
||||||
my $cacheInfoFile = "$destDir/nix-cache-info";
|
|
||||||
if (! -e $cacheInfoFile) {
|
|
||||||
open FILE, ">$cacheInfoFile" or die "cannot create $cacheInfoFile: $!";
|
|
||||||
print FILE "StoreDir: $Nix::Config::storeDir\n";
|
|
||||||
print FILE "WantMassQuery: 0\n"; # by default, don't hit this cache for "nix-env -qas"
|
|
||||||
close FILE;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
# Copy the archives and the corresponding NAR info files.
|
|
||||||
print STDERR "copying archives...\n";
|
|
||||||
|
|
||||||
my $totalNarSize = 0;
|
|
||||||
my $totalCompressedSize = 0;
|
|
||||||
|
|
||||||
for (my $n = 0; $n < scalar @storePaths2; $n++) {
|
|
||||||
my $storePath = $storePaths2[$n];
|
|
||||||
my $narDir = $narPaths[$n];
|
|
||||||
my $baseName = basename $storePath;
|
|
||||||
|
|
||||||
# Get info about the store path.
|
|
||||||
my ($deriver, $narHash, $time, $narSize, $refs) = queryPathInfo($storePath, 1);
|
|
||||||
|
|
||||||
# In some exceptional cases (such as VM tests that use the Nix
|
|
||||||
# store of the host), the database doesn't contain the hash. So
|
|
||||||
# compute it.
|
|
||||||
if ($narHash =~ /^sha256:0*$/) {
|
|
||||||
my $nar = "$tmpDir/nar";
|
|
||||||
system("$Nix::Config::binDir/nix-store --dump $storePath > $nar") == 0
|
|
||||||
or die "cannot dump $storePath\n";
|
|
||||||
$narHash = `$Nix::Config::binDir/nix-hash --type sha256 --base32 --flat $nar`;
|
|
||||||
die "cannot hash ‘$nar’" if $? != 0;
|
|
||||||
chomp $narHash;
|
|
||||||
$narHash = "sha256:$narHash";
|
|
||||||
$narSize = stat("$nar")->size;
|
|
||||||
unlink $nar or die;
|
|
||||||
}
|
|
||||||
|
|
||||||
$totalNarSize += $narSize;
|
|
||||||
|
|
||||||
# Get info about the compressed NAR.
|
|
||||||
open HASH, "$narDir/nar-compressed-hash" or die "cannot open nar-compressed-hash";
|
|
||||||
my $compressedHash = <HASH>;
|
|
||||||
chomp $compressedHash;
|
|
||||||
$compressedHash =~ /^[0-9a-z]+$/ or die "invalid hash";
|
|
||||||
close HASH;
|
|
||||||
|
|
||||||
my $narName = "$compressedHash.nar" . ($compressionType eq "xz" ? ".xz" : $compressionType eq "bzip2" ? ".bz2" : "");
|
|
||||||
|
|
||||||
my $narFile = "$narDir/$narName";
|
|
||||||
(-f $narFile) or die "NAR file for $storePath not found";
|
|
||||||
|
|
||||||
my $compressedSize = stat($narFile)->size;
|
|
||||||
$totalCompressedSize += $compressedSize;
|
|
||||||
|
|
||||||
printf STDERR "%s [%.2f MiB, %.1f%%]\n", $storePath,
|
|
||||||
$compressedSize / (1024 * 1024), $compressedSize / $narSize * 100;
|
|
||||||
|
|
||||||
# Copy the compressed NAR.
|
|
||||||
my $dst = "$destDir/$narName";
|
|
||||||
if (! -f $dst) {
|
|
||||||
my $tmp = "$destDir/.tmp.$$.$narName";
|
|
||||||
if ($link) {
|
|
||||||
link($narFile, $tmp) or die "cannot link $tmp to $narFile: $!\n";
|
|
||||||
} else {
|
|
||||||
copy($narFile, $tmp) or die "cannot copy $narFile to $tmp: $!\n";
|
|
||||||
}
|
|
||||||
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
# Write the info file.
|
|
||||||
my $info;
|
|
||||||
$info .= "StorePath: $storePath\n";
|
|
||||||
$info .= "URL: $narName\n";
|
|
||||||
$info .= "Compression: $compressionType\n";
|
|
||||||
$info .= "FileHash: sha256:$compressedHash\n";
|
|
||||||
$info .= "FileSize: $compressedSize\n";
|
|
||||||
$info .= "NarHash: $narHash\n";
|
|
||||||
$info .= "NarSize: $narSize\n";
|
|
||||||
$info .= "References: " . join(" ", map { basename $_ } @{$refs}) . "\n";
|
|
||||||
if (defined $deriver) {
|
|
||||||
$info .= "Deriver: " . basename $deriver . "\n";
|
|
||||||
if (isValidPath($deriver)) {
|
|
||||||
my $drv = derivationFromPath($deriver);
|
|
||||||
$info .= "System: $drv->{platform}\n";
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (defined $secretKeyFile) {
|
|
||||||
my $secretKey = readFile $secretKeyFile;
|
|
||||||
my $fingerprint = fingerprintPath($storePath, $narHash, $narSize, $refs);
|
|
||||||
my $sig = signString($secretKey, $fingerprint);
|
|
||||||
$info .= "Sig: $sig\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
my $pathHash = substr(basename($storePath), 0, 32);
|
|
||||||
|
|
||||||
$dst = "$destDir/$pathHash.narinfo";
|
|
||||||
if ($force || ! -f $dst) {
|
|
||||||
my $tmp = "$destDir/.tmp.$$.$pathHash.narinfo";
|
|
||||||
open INFO, ">$tmp" or die;
|
|
||||||
print INFO "$info" or die;
|
|
||||||
close INFO or die;
|
|
||||||
rename($tmp, $dst) or die "cannot rename $tmp to $dst: $!\n";
|
|
||||||
}
|
|
||||||
|
|
||||||
$narFiles{$storePath} = [
|
|
||||||
{ url => "$archivesURL/$narName"
|
|
||||||
, hash => "sha256:$compressedHash"
|
|
||||||
, size => $compressedSize
|
|
||||||
, compressionType => $compressionType
|
|
||||||
, narHash => "$narHash"
|
|
||||||
, narSize => $narSize
|
|
||||||
, references => join(" ", @{$refs})
|
|
||||||
, deriver => $deriver
|
|
||||||
}
|
|
||||||
] if $writeManifest;
|
|
||||||
}
|
|
||||||
|
|
||||||
printf STDERR "total compressed size %.2f MiB, %.1f%%\n",
|
|
||||||
$totalCompressedSize / (1024 * 1024), $totalCompressedSize / ($totalNarSize || 1) * 100;
|
|
||||||
|
|
||||||
|
|
||||||
# Optionally write a manifest.
|
|
||||||
writeManifest($manifestPath // "$destDir/MANIFEST", \%narFiles, \()) if $writeManifest;
|
|
||||||
|
|
@ -6,7 +6,7 @@ clearCache
|
||||||
# Create the binary cache.
|
# Create the binary cache.
|
||||||
outPath=$(nix-build dependencies.nix --no-out-link)
|
outPath=$(nix-build dependencies.nix --no-out-link)
|
||||||
|
|
||||||
nix-push --dest $cacheDir $outPath
|
nix copy --recursive --to file://$cacheDir $outPath
|
||||||
|
|
||||||
|
|
||||||
basicTests() {
|
basicTests() {
|
||||||
|
|
@ -58,7 +58,7 @@ unset _NIX_FORCE_HTTP_BINARY_CACHE_STORE
|
||||||
# Test whether Nix notices if the NAR doesn't match the hash in the NAR info.
|
# Test whether Nix notices if the NAR doesn't match the hash in the NAR info.
|
||||||
clearStore
|
clearStore
|
||||||
|
|
||||||
nar=$(ls $cacheDir/*.nar.xz | head -n1)
|
nar=$(ls $cacheDir/nar/*.nar.xz | head -n1)
|
||||||
mv $nar $nar.good
|
mv $nar $nar.good
|
||||||
mkdir -p $TEST_ROOT/empty
|
mkdir -p $TEST_ROOT/empty
|
||||||
nix-store --dump $TEST_ROOT/empty | xz > $nar
|
nix-store --dump $TEST_ROOT/empty | xz > $nar
|
||||||
|
|
@ -117,7 +117,7 @@ badKey="$(cat $TEST_ROOT/pk2)"
|
||||||
res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
|
res=($(nix-store --generate-binary-cache-key foo.nixos.org-1 $TEST_ROOT/sk3 $TEST_ROOT/pk3))
|
||||||
otherKey="$(cat $TEST_ROOT/pk3)"
|
otherKey="$(cat $TEST_ROOT/pk3)"
|
||||||
|
|
||||||
nix-push --dest $cacheDir --key-file $TEST_ROOT/sk1 $outPath
|
nix copy --recursive --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath
|
||||||
|
|
||||||
|
|
||||||
# Downloading should fail if we don't provide a key.
|
# Downloading should fail if we don't provide a key.
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ check:
|
||||||
|
|
||||||
nix_tests = \
|
nix_tests = \
|
||||||
init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
init.sh hash.sh lang.sh add.sh simple.sh dependencies.sh \
|
||||||
build-hook.sh nix-push.sh gc.sh gc-concurrent.sh \
|
build-hook.sh gc.sh gc-concurrent.sh \
|
||||||
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
|
referrers.sh user-envs.sh logging.sh nix-build.sh misc.sh fixed.sh \
|
||||||
gc-runtime.sh check-refs.sh filter-source.sh \
|
gc-runtime.sh check-refs.sh filter-source.sh \
|
||||||
remote-store.sh export.sh export-graph.sh \
|
remote-store.sh export.sh export-graph.sh \
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,7 @@ nix-channel --remove xyzzy
|
||||||
# Create a channel.
|
# Create a channel.
|
||||||
rm -rf $TEST_ROOT/foo
|
rm -rf $TEST_ROOT/foo
|
||||||
mkdir -p $TEST_ROOT/foo
|
mkdir -p $TEST_ROOT/foo
|
||||||
nix-push --dest $TEST_ROOT/foo --manifest --bzip2 $(nix-store -r $(nix-instantiate dependencies.nix))
|
nix copy --recursive --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r $(nix-instantiate dependencies.nix))
|
||||||
rm -rf $TEST_ROOT/nixexprs
|
rm -rf $TEST_ROOT/nixexprs
|
||||||
mkdir -p $TEST_ROOT/nixexprs
|
mkdir -p $TEST_ROOT/nixexprs
|
||||||
cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/
|
cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/
|
||||||
|
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
source common.sh
|
|
||||||
|
|
||||||
clearStore
|
|
||||||
|
|
||||||
drvPath=$(nix-instantiate dependencies.nix)
|
|
||||||
outPath=$(nix-store -r $drvPath)
|
|
||||||
|
|
||||||
echo "pushing $drvPath"
|
|
||||||
|
|
||||||
mkdir -p $TEST_ROOT/cache
|
|
||||||
|
|
||||||
nix-push --dest $TEST_ROOT/cache --manifest $drvPath --bzip2
|
|
||||||
|
|
@ -46,7 +46,7 @@ fi
|
||||||
# --verify can fix it.
|
# --verify can fix it.
|
||||||
clearCache
|
clearCache
|
||||||
|
|
||||||
nix-push --dest $cacheDir $path
|
nix copy --recursive --to file://$cacheDir $path
|
||||||
|
|
||||||
chmod u+w $path2
|
chmod u+w $path2
|
||||||
rm -rf $path2
|
rm -rf $path2
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue