diff --git a/third_party/nix/.dir-locals.el b/third_party/nix/.dir-locals.el new file mode 100644 index 000000000..2d1117f4b --- /dev/null +++ b/third_party/nix/.dir-locals.el @@ -0,0 +1,16 @@ +((c++-mode . ( + (c-file-style . "k&r") + (c-basic-offset . 4) + (indent-tabs-mode . nil) + (tab-width . 4) + (show-trailing-whitespace . t) + (indicate-empty-lines . t) + (eval . (c-set-offset 'innamespace 0)) + (eval . (c-set-offset 'defun-open 0)) + (eval . (c-set-offset 'inline-open 0)) + (eval . (c-set-offset 'arglist-intro '+)) + (eval . (c-set-offset 'arglist-cont 0)) + (eval . (c-set-offset 'arglist-cont-nonempty '+)) + (eval . (c-set-offset 'substatement-open 0)) + (eval . (c-set-offset 'access-label '-)) + ))) diff --git a/third_party/nix/.editorconfig b/third_party/nix/.editorconfig new file mode 100644 index 000000000..887ecadba --- /dev/null +++ b/third_party/nix/.editorconfig @@ -0,0 +1,26 @@ +# EditorConfig configuration for nix +# http://EditorConfig.org + +# Top-most EditorConfig file +root = true + +# Unix-style newlines with a newline ending every file, utf-8 charset +[*] +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +charset = utf-8 + +# Match nix files, set indent to spaces with width of two +[*.nix] +indent_style = space +indent_size = 2 + +# Match c++/shell/perl, set indent to spaces with width of four +[*.{hpp,cc,hh,sh,pl}] +indent_style = space +indent_size = 4 + +# Match diffs, avoid to trim trailing whitespace +[*.{diff,patch}] +trim_trailing_whitespace = false diff --git a/third_party/nix/.github/ISSUE_TEMPLATE.md b/third_party/nix/.github/ISSUE_TEMPLATE.md new file mode 100644 index 000000000..3372b1f03 --- /dev/null +++ b/third_party/nix/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,27 @@ + diff --git a/third_party/nix/.gitignore b/third_party/nix/.gitignore new file mode 100644 index 000000000..b75c5d489 --- /dev/null +++ b/third_party/nix/.gitignore @@ -0,0 +1,119 @@ +Makefile.config +perl/Makefile.config + +# / +/aclocal.m4 +/autom4te.cache +/config.* +/configure +/nix.spec +/stamp-h1 +/svn-revision +/libtool + +/corepkgs/config.nix + +# /corepkgs/channels/ +/corepkgs/channels/unpack.sh + +# /corepkgs/nar/ +/corepkgs/nar/nar.sh +/corepkgs/nar/unnar.sh + +# /doc/manual/ +/doc/manual/manual.html +/doc/manual/manual.xmli +/doc/manual/manual.pdf +/doc/manual/manual.is-valid +/doc/manual/*.1 +/doc/manual/*.5 +/doc/manual/*.8 +/doc/manual/version.txt + +# /scripts/ +/scripts/nix-profile.sh +/scripts/nix-copy-closure +/scripts/nix-reduce-build +/scripts/nix-http-export.cgi +/scripts/nix-profile-daemon.sh + +# /src/libexpr/ +/src/libexpr/lexer-tab.cc +/src/libexpr/lexer-tab.hh +/src/libexpr/parser-tab.cc +/src/libexpr/parser-tab.hh +/src/libexpr/parser-tab.output +/src/libexpr/nix.tbl + +# /src/libstore/ +/src/libstore/*.gen.hh + +/src/nix/nix + +# /src/nix-env/ +/src/nix-env/nix-env + +# /src/nix-instantiate/ +/src/nix-instantiate/nix-instantiate + +# /src/nix-store/ +/src/nix-store/nix-store + +/src/nix-prefetch-url/nix-prefetch-url + +# /src/nix-daemon/ +/src/nix-daemon/nix-daemon + +/src/nix-collect-garbage/nix-collect-garbage + +# /src/nix-channel/ +/src/nix-channel/nix-channel + +# /src/nix-build/ +/src/nix-build/nix-build + +/src/nix-copy-closure/nix-copy-closure + +/src/build-remote/build-remote + +# /tests/ +/tests/test-tmp +/tests/common.sh +/tests/dummy +/tests/result* +/tests/restricted-innocent +/tests/shell +/tests/shell.drv + +# /tests/lang/ +/tests/lang/*.out +/tests/lang/*.out.xml +/tests/lang/*.ast + +/perl/lib/Nix/Config.pm +/perl/lib/Nix/Store.cc + +/misc/systemd/nix-daemon.service +/misc/systemd/nix-daemon.socket +/misc/upstart/nix-daemon.conf + +/src/resolve-system-dependencies/resolve-system-dependencies + +inst/ + +*.a +*.o +*.so +*.dylib +*.dll +*.exe +*.dep +*~ +*.pc +*.plist + +# GNU Global +GPATH +GRTAGS +GSYMS +GTAGS diff --git a/third_party/nix/.travis.yml b/third_party/nix/.travis.yml new file mode 100644 index 000000000..99218a963 --- /dev/null +++ b/third_party/nix/.travis.yml @@ -0,0 +1,2 @@ +os: osx +script: ./tests/install-darwin.sh diff --git a/third_party/nix/.version b/third_party/nix/.version new file mode 100644 index 000000000..fd06a9268 --- /dev/null +++ b/third_party/nix/.version @@ -0,0 +1 @@ +2.3.4 \ No newline at end of file diff --git a/third_party/nix/COPYING b/third_party/nix/COPYING new file mode 100644 index 000000000..5ab7695ab --- /dev/null +++ b/third_party/nix/COPYING @@ -0,0 +1,504 @@ + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 + + Copyright (C) 1991, 1999 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + +[This is the first released version of the Lesser GPL. It also counts + as the successor of the GNU Library Public License, version 2, hence + the version number 2.1.] + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +Licenses are intended to guarantee your freedom to share and change +free software--to make sure the software is free for all its users. + + This license, the Lesser General Public License, applies to some +specially designated software packages--typically libraries--of the +Free Software Foundation and other authors who decide to use it. You +can use it too, but we suggest you first think carefully about whether +this license or the ordinary General Public License is the better +strategy to use in any particular case, based on the explanations below. + + When we speak of free software, we are referring to freedom of use, +not price. Our General Public Licenses are designed to make sure that +you have the freedom to distribute copies of free software (and charge +for this service if you wish); that you receive source code or can get +it if you want it; that you can change the software and use pieces of +it in new free programs; and that you are informed that you can do +these things. + + To protect your rights, we need to make restrictions that forbid +distributors to deny you these rights or to ask you to surrender these +rights. These restrictions translate to certain responsibilities for +you if you distribute copies of the library or if you modify it. + + For example, if you distribute copies of the library, whether gratis +or for a fee, you must give the recipients all the rights that we gave +you. You must make sure that they, too, receive or can get the source +code. If you link other code with the library, you must provide +complete object files to the recipients, so that they can relink them +with the library after making changes to the library and recompiling +it. And you must show them these terms so they know their rights. + + We protect your rights with a two-step method: (1) we copyright the +library, and (2) we offer you this license, which gives you legal +permission to copy, distribute and/or modify the library. + + To protect each distributor, we want to make it very clear that +there is no warranty for the free library. Also, if the library is +modified by someone else and passed on, the recipients should know +that what they have is not the original version, so that the original +author's reputation will not be affected by problems that might be +introduced by others. + + Finally, software patents pose a constant threat to the existence of +any free program. We wish to make sure that a company cannot +effectively restrict the users of a free program by obtaining a +restrictive license from a patent holder. Therefore, we insist that +any patent license obtained for a version of the library must be +consistent with the full freedom of use specified in this license. + + Most GNU software, including some libraries, is covered by the +ordinary GNU General Public License. This license, the GNU Lesser +General Public License, applies to certain designated libraries, and +is quite different from the ordinary General Public License. We use +this license for certain libraries in order to permit linking those +libraries into non-free programs. + + When a program is linked with a library, whether statically or using +a shared library, the combination of the two is legally speaking a +combined work, a derivative of the original library. The ordinary +General Public License therefore permits such linking only if the +entire combination fits its criteria of freedom. The Lesser General +Public License permits more lax criteria for linking other code with +the library. + + We call this license the "Lesser" General Public License because it +does Less to protect the user's freedom than the ordinary General +Public License. It also provides other free software developers Less +of an advantage over competing non-free programs. These disadvantages +are the reason we use the ordinary General Public License for many +libraries. However, the Lesser license provides advantages in certain +special circumstances. + + For example, on rare occasions, there may be a special need to +encourage the widest possible use of a certain library, so that it becomes +a de-facto standard. To achieve this, non-free programs must be +allowed to use the library. A more frequent case is that a free +library does the same job as widely used non-free libraries. In this +case, there is little to gain by limiting the free library to free +software only, so we use the Lesser General Public License. + + In other cases, permission to use a particular library in non-free +programs enables a greater number of people to use a large body of +free software. For example, permission to use the GNU C Library in +non-free programs enables many more people to use the whole GNU +operating system, as well as its variant, the GNU/Linux operating +system. + + Although the Lesser General Public License is Less protective of the +users' freedom, it does ensure that the user of a program that is +linked with the Library has the freedom and the wherewithal to run +that program using a modified version of the Library. + + The precise terms and conditions for copying, distribution and +modification follow. Pay close attention to the difference between a +"work based on the library" and a "work that uses the library". The +former contains code derived from the library, whereas the latter must +be combined with the library in order to run. + + GNU LESSER GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License Agreement applies to any software library or other +program which contains a notice placed by the copyright holder or +other authorized party saying it may be distributed under the terms of +this Lesser General Public License (also called "this License"). +Each licensee is addressed as "you". + + A "library" means a collection of software functions and/or data +prepared so as to be conveniently linked with application programs +(which use some of those functions and data) to form executables. + + The "Library", below, refers to any such software library or work +which has been distributed under these terms. A "work based on the +Library" means either the Library or any derivative work under +copyright law: that is to say, a work containing the Library or a +portion of it, either verbatim or with modifications and/or translated +straightforwardly into another language. (Hereinafter, translation is +included without limitation in the term "modification".) + + "Source code" for a work means the preferred form of the work for +making modifications to it. For a library, complete source code means +all the source code for all modules it contains, plus any associated +interface definition files, plus the scripts used to control compilation +and installation of the library. + + Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running a program using the Library is not restricted, and output from +such a program is covered only if its contents constitute a work based +on the Library (independent of the use of the Library in a tool for +writing it). Whether that is true depends on what the Library does +and what the program that uses the Library does. + + 1. You may copy and distribute verbatim copies of the Library's +complete source code as you receive it, in any medium, provided that +you conspicuously and appropriately publish on each copy an +appropriate copyright notice and disclaimer of warranty; keep intact +all the notices that refer to this License and to the absence of any +warranty; and distribute a copy of this License along with the +Library. + + You may charge a fee for the physical act of transferring a copy, +and you may at your option offer warranty protection in exchange for a +fee. + + 2. You may modify your copy or copies of the Library or any portion +of it, thus forming a work based on the Library, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) The modified work must itself be a software library. + + b) You must cause the files modified to carry prominent notices + stating that you changed the files and the date of any change. + + c) You must cause the whole of the work to be licensed at no + charge to all third parties under the terms of this License. + + d) If a facility in the modified Library refers to a function or a + table of data to be supplied by an application program that uses + the facility, other than as an argument passed when the facility + is invoked, then you must make a good faith effort to ensure that, + in the event an application does not supply such function or + table, the facility still operates, and performs whatever part of + its purpose remains meaningful. + + (For example, a function in a library to compute square roots has + a purpose that is entirely well-defined independent of the + application. Therefore, Subsection 2d requires that any + application-supplied function or table used by this function must + be optional: if the application does not supply it, the square + root function must still compute square roots.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Library, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Library, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote +it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Library. + +In addition, mere aggregation of another work not based on the Library +with the Library (or with a work based on the Library) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may opt to apply the terms of the ordinary GNU General Public +License instead of this License to a given copy of the Library. To do +this, you must alter all the notices that refer to this License, so +that they refer to the ordinary GNU General Public License, version 2, +instead of to this License. (If a newer version than version 2 of the +ordinary GNU General Public License has appeared, then you can specify +that version instead if you wish.) Do not make any other change in +these notices. + + Once this change is made in a given copy, it is irreversible for +that copy, so the ordinary GNU General Public License applies to all +subsequent copies and derivative works made from that copy. + + This option is useful when you wish to copy part of the code of +the Library into a program that is not a library. + + 4. You may copy and distribute the Library (or a portion or +derivative of it, under Section 2) in object code or executable form +under the terms of Sections 1 and 2 above provided that you accompany +it with the complete corresponding machine-readable source code, which +must be distributed under the terms of Sections 1 and 2 above on a +medium customarily used for software interchange. + + If distribution of object code is made by offering access to copy +from a designated place, then offering equivalent access to copy the +source code from the same place satisfies the requirement to +distribute the source code, even though third parties are not +compelled to copy the source along with the object code. + + 5. A program that contains no derivative of any portion of the +Library, but is designed to work with the Library by being compiled or +linked with it, is called a "work that uses the Library". Such a +work, in isolation, is not a derivative work of the Library, and +therefore falls outside the scope of this License. + + However, linking a "work that uses the Library" with the Library +creates an executable that is a derivative of the Library (because it +contains portions of the Library), rather than a "work that uses the +library". The executable is therefore covered by this License. +Section 6 states terms for distribution of such executables. + + When a "work that uses the Library" uses material from a header file +that is part of the Library, the object code for the work may be a +derivative work of the Library even though the source code is not. +Whether this is true is especially significant if the work can be +linked without the Library, or if the work is itself a library. The +threshold for this to be true is not precisely defined by law. + + If such an object file uses only numerical parameters, data +structure layouts and accessors, and small macros and small inline +functions (ten lines or less in length), then the use of the object +file is unrestricted, regardless of whether it is legally a derivative +work. (Executables containing this object code plus portions of the +Library will still fall under Section 6.) + + Otherwise, if the work is a derivative of the Library, you may +distribute the object code for the work under the terms of Section 6. +Any executables containing that work also fall under Section 6, +whether or not they are linked directly with the Library itself. + + 6. As an exception to the Sections above, you may also combine or +link a "work that uses the Library" with the Library to produce a +work containing portions of the Library, and distribute that work +under terms of your choice, provided that the terms permit +modification of the work for the customer's own use and reverse +engineering for debugging such modifications. + + You must give prominent notice with each copy of the work that the +Library is used in it and that the Library and its use are covered by +this License. You must supply a copy of this License. If the work +during execution displays copyright notices, you must include the +copyright notice for the Library among them, as well as a reference +directing the user to the copy of this License. Also, you must do one +of these things: + + a) Accompany the work with the complete corresponding + machine-readable source code for the Library including whatever + changes were used in the work (which must be distributed under + Sections 1 and 2 above); and, if the work is an executable linked + with the Library, with the complete machine-readable "work that + uses the Library", as object code and/or source code, so that the + user can modify the Library and then relink to produce a modified + executable containing the modified Library. (It is understood + that the user who changes the contents of definitions files in the + Library will not necessarily be able to recompile the application + to use the modified definitions.) + + b) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (1) uses at run time a + copy of the library already present on the user's computer system, + rather than copying library functions into the executable, and (2) + will operate properly with a modified version of the library, if + the user installs one, as long as the modified version is + interface-compatible with the version that the work was made with. + + c) Accompany the work with a written offer, valid for at + least three years, to give the same user the materials + specified in Subsection 6a, above, for a charge no more + than the cost of performing this distribution. + + d) If distribution of the work is made by offering access to copy + from a designated place, offer equivalent access to copy the above + specified materials from the same place. + + e) Verify that the user has already received a copy of these + materials or that you have already sent this user a copy. + + For an executable, the required form of the "work that uses the +Library" must include any data and utility programs needed for +reproducing the executable from it. However, as a special exception, +the materials to be distributed need not include anything that is +normally distributed (in either source or binary form) with the major +components (compiler, kernel, and so on) of the operating system on +which the executable runs, unless that component itself accompanies +the executable. + + It may happen that this requirement contradicts the license +restrictions of other proprietary libraries that do not normally +accompany the operating system. Such a contradiction means you cannot +use both them and the Library together in an executable that you +distribute. + + 7. You may place library facilities that are a work based on the +Library side-by-side in a single library together with other library +facilities not covered by this License, and distribute such a combined +library, provided that the separate distribution of the work based on +the Library and of the other library facilities is otherwise +permitted, and provided that you do these two things: + + a) Accompany the combined library with a copy of the same work + based on the Library, uncombined with any other library + facilities. This must be distributed under the terms of the + Sections above. + + b) Give prominent notice with the combined library of the fact + that part of it is a work based on the Library, and explaining + where to find the accompanying uncombined form of the same work. + + 8. You may not copy, modify, sublicense, link with, or distribute +the Library except as expressly provided under this License. Any +attempt otherwise to copy, modify, sublicense, link with, or +distribute the Library is void, and will automatically terminate your +rights under this License. However, parties who have received copies, +or rights, from you under this License will not have their licenses +terminated so long as such parties remain in full compliance. + + 9. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Library or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Library (or any work based on the +Library), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Library or works based on it. + + 10. Each time you redistribute the Library (or any work based on the +Library), the recipient automatically receives a license from the +original licensor to copy, distribute, link with or modify the Library +subject to these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties with +this License. + + 11. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Library at all. For example, if a patent +license would not permit royalty-free redistribution of the Library by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Library. + +If any portion of this section is held invalid or unenforceable under any +particular circumstance, the balance of the section is intended to apply, +and the section as a whole is intended to apply in other circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 12. If the distribution and/or use of the Library is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Library under this License may add +an explicit geographical distribution limitation excluding those countries, +so that distribution is permitted only in or among countries not thus +excluded. In such case, this License incorporates the limitation as if +written in the body of this License. + + 13. The Free Software Foundation may publish revised and/or new +versions of the Lesser General Public License from time to time. +Such new versions will be similar in spirit to the present version, +but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Library +specifies a version number of this License which applies to it and +"any later version", you have the option of following the terms and +conditions either of that version or of any later version published by +the Free Software Foundation. If the Library does not specify a +license version number, you may choose any version ever published by +the Free Software Foundation. + + 14. If you wish to incorporate parts of the Library into other free +programs whose distribution conditions are incompatible with these, +write to the author to ask for permission. For software which is +copyrighted by the Free Software Foundation, write to the Free +Software Foundation; we sometimes make exceptions for this. Our +decision will be guided by the two goals of preserving the free status +of all derivatives of our free software and of promoting the sharing +and reuse of software generally. + + NO WARRANTY + + 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO +WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. +EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR +OTHER PARTIES PROVIDE THE LIBRARY "AS IS" WITHOUT WARRANTY OF ANY +KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE +LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, YOU ASSUME +THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN +WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY +AND/OR REDISTRIBUTE THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU +FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR +CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE +LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING +RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A +FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF +SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Libraries + + If you develop a new library, and you want it to be of the greatest +possible use to the public, we recommend making it free software that +everyone can redistribute and change. You can do so by permitting +redistribution under these terms (or, alternatively, under the terms of the +ordinary General Public License). + + To apply these terms, attach the following notices to the library. It is +safest to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least the +"copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +Also add information on how to contact you by electronic and paper mail. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the library, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the + library `Frob' (a library for tweaking knobs) written by James Random Hacker. + + , 1 April 1990 + Ty Coon, President of Vice + +That's all there is to it! + + diff --git a/third_party/nix/Makefile b/third_party/nix/Makefile new file mode 100644 index 000000000..9ac82fda6 --- /dev/null +++ b/third_party/nix/Makefile @@ -0,0 +1,22 @@ +makefiles = \ + local.mk \ + src/libutil/local.mk \ + src/libstore/local.mk \ + src/libmain/local.mk \ + src/libexpr/local.mk \ + src/nix/local.mk \ + src/resolve-system-dependencies/local.mk \ + scripts/local.mk \ + corepkgs/local.mk \ + misc/systemd/local.mk \ + misc/launchd/local.mk \ + misc/upstart/local.mk \ + doc/manual/local.mk \ + tests/local.mk \ + tests/plugins/local.mk + +GLOBAL_CXXFLAGS += -g -Wall -include config.h + +-include Makefile.config + +include mk/lib.mk diff --git a/third_party/nix/Makefile.config.in b/third_party/nix/Makefile.config.in new file mode 100644 index 000000000..7e3b35b98 --- /dev/null +++ b/third_party/nix/Makefile.config.in @@ -0,0 +1,41 @@ +AR = @AR@ +BDW_GC_LIBS = @BDW_GC_LIBS@ +BUILD_SHARED_LIBS = @BUILD_SHARED_LIBS@ +CC = @CC@ +CFLAGS = @CFLAGS@ +CXX = @CXX@ +CXXFLAGS = @CXXFLAGS@ +LDFLAGS = @LDFLAGS@ +ENABLE_S3 = @ENABLE_S3@ +HAVE_SODIUM = @HAVE_SODIUM@ +HAVE_SECCOMP = @HAVE_SECCOMP@ +BOOST_LDFLAGS = @BOOST_LDFLAGS@ +LIBCURL_LIBS = @LIBCURL_LIBS@ +OPENSSL_LIBS = @OPENSSL_LIBS@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +SODIUM_LIBS = @SODIUM_LIBS@ +LIBLZMA_LIBS = @LIBLZMA_LIBS@ +SQLITE3_LIBS = @SQLITE3_LIBS@ +LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ +EDITLINE_LIBS = @EDITLINE_LIBS@ +bash = @bash@ +bindir = @bindir@ +lsof = @lsof@ +datadir = @datadir@ +datarootdir = @datarootdir@ +docdir = @docdir@ +exec_prefix = @exec_prefix@ +includedir = @includedir@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +pkglibdir = $(libdir)/$(PACKAGE_NAME) +prefix = @prefix@ +sandbox_shell = @sandbox_shell@ +storedir = @storedir@ +sysconfdir = @sysconfdir@ +doc_generate = @doc_generate@ +xmllint = @xmllint@ +xsltproc = @xsltproc@ diff --git a/third_party/nix/README.md b/third_party/nix/README.md new file mode 100644 index 000000000..48cb1685c --- /dev/null +++ b/third_party/nix/README.md @@ -0,0 +1,24 @@ +[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) + +Nix, the purely functional package manager +------------------------------------------ + +Nix is a new take on package management that is fairly unique. Because of its +purity aspects, a lot of issues found in traditional package managers don't +appear with Nix. + +To find out more about the tool, usage and installation instructions, please +read the manual, which is available on the Nix website at +. + +## Contributing + +Take a look at the [Hacking Section](http://nixos.org/nix/manual/#chap-hacking) +of the manual. It helps you to get started with building Nix from source. + +## License + +Nix is released under the LGPL v2.1 + +This product includes software developed by the OpenSSL Project for +use in the [OpenSSL Toolkit](http://www.OpenSSL.org/). diff --git a/third_party/nix/bootstrap.sh b/third_party/nix/bootstrap.sh new file mode 100755 index 000000000..e3e259351 --- /dev/null +++ b/third_party/nix/bootstrap.sh @@ -0,0 +1,4 @@ +#! /bin/sh -e +rm -f aclocal.m4 +mkdir -p config +exec autoreconf -vfi diff --git a/third_party/nix/config/config.guess b/third_party/nix/config/config.guess new file mode 100755 index 000000000..d4fb3213e --- /dev/null +++ b/third_party/nix/config/config.guess @@ -0,0 +1,1486 @@ +#! /bin/sh +# Attempt to guess a canonical system name. +# Copyright 1992-2018 Free Software Foundation, Inc. + +timestamp='2018-08-02' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). +# +# Originally written by Per Bothner; maintained since 2000 by Ben Elliston. +# +# You can get the latest version of this script from: +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess +# +# Please send patches to . + + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] + +Output the configuration name of the system \`$me' is run on. + +Options: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.guess ($timestamp) + +Originally written by Per Bothner. +Copyright 1992-2018 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" >&2 + exit 1 ;; + * ) + break ;; + esac +done + +if test $# != 0; then + echo "$me: too many arguments$help" >&2 + exit 1 +fi + +# CC_FOR_BUILD -- compiler used by this script. Note that the use of a +# compiler to aid in system detection is discouraged as it requires +# temporary files to be created and, as you can see below, it is a +# headache to deal with in a portable fashion. + +# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still +# use `HOST_CC' if defined, but it is deprecated. + +# Portable tmp directory creation inspired by the Autoconf team. + +tmp= +# shellcheck disable=SC2172 +trap 'test -z "$tmp" || rm -fr "$tmp"' 1 2 13 15 +trap 'exitcode=$?; test -z "$tmp" || rm -fr "$tmp"; exit $exitcode' 0 + +set_cc_for_build() { + : "${TMPDIR=/tmp}" + # shellcheck disable=SC2039 + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir "$tmp" 2>/dev/null) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir "$tmp" 2>/dev/null) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } + dummy=$tmp/dummy + case ${CC_FOR_BUILD-},${HOST_CC-},${CC-} in + ,,) echo "int x;" > "$dummy.c" + for driver in cc gcc c89 c99 ; do + if ($driver -c -o "$dummy.o" "$dummy.c") >/dev/null 2>&1 ; then + CC_FOR_BUILD="$driver" + break + fi + done + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; + esac +} + +# This is needed to find uname on a Pyramid OSx when run in the BSD universe. +# (ghazi@noc.rutgers.edu 1994-08-24) +if test -f /.attbin/uname ; then + PATH=$PATH:/.attbin ; export PATH +fi + +UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown +UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown +UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown +UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown + +case "$UNAME_SYSTEM" in +Linux|GNU|GNU/*) + # If the system lacks a compiler, then just pick glibc. + # We could probably try harder. + LIBC=gnu + + set_cc_for_build + cat <<-EOF > "$dummy.c" + #include + #if defined(__UCLIBC__) + LIBC=uclibc + #elif defined(__dietlibc__) + LIBC=dietlibc + #else + LIBC=gnu + #endif + EOF + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^LIBC' | sed 's, ,,g'`" + + # If ldd exists, use it to detect musl libc. + if command -v ldd >/dev/null && \ + ldd --version 2>&1 | grep -q ^musl + then + LIBC=musl + fi + ;; +esac + +# Note: order is significant - the case branches are not exclusive. + +case "$UNAME_MACHINE:$UNAME_SYSTEM:$UNAME_RELEASE:$UNAME_VERSION" in + *:NetBSD:*:*) + # NetBSD (nbsd) targets should (where applicable) match one or + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, + # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently + # switched to ELF, *-*-netbsd* would select the old + # object file format. This provides both forward + # compatibility and a consistent mechanism for selecting the + # object file format. + # + # Note: NetBSD doesn't particularly care about the vendor + # portion of the name. We always set it to "unknown". + sysctl="sysctl -n hw.machine_arch" + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + "/sbin/$sysctl" 2>/dev/null || \ + "/usr/sbin/$sysctl" 2>/dev/null || \ + echo unknown)` + case "$UNAME_MACHINE_ARCH" in + armeb) machine=armeb-unknown ;; + arm*) machine=arm-unknown ;; + sh3el) machine=shl-unknown ;; + sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; + earmv*) + arch=`echo "$UNAME_MACHINE_ARCH" | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo "$UNAME_MACHINE_ARCH" | sed -ne 's,^.*\(eb\)$,\1,p'` + machine="${arch}${endian}"-unknown + ;; + *) machine="$UNAME_MACHINE_ARCH"-unknown ;; + esac + # The Operating System including object format, if it has switched + # to ELF recently (or will in the future) and ABI. + case "$UNAME_MACHINE_ARCH" in + earm*) + os=netbsdelf + ;; + arm*|i386|m68k|ns32k|sh3*|sparc|vax) + set_cc_for_build + if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ELF__ + then + # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). + # Return netbsd for either. FIX? + os=netbsd + else + os=netbsdelf + fi + ;; + *) + os=netbsd + ;; + esac + # Determine ABI tags. + case "$UNAME_MACHINE_ARCH" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo "$UNAME_MACHINE_ARCH" | sed -e "$expr"` + ;; + esac + # The OS release + # Debian GNU/NetBSD machines have a different userland, and + # thus, need a distinct triplet. However, they do not need + # kernel version information, so it can be replaced with a + # suitable tag, in the style of linux-gnu. + case "$UNAME_VERSION" in + Debian*) + release='-gnu' + ;; + *) + release=`echo "$UNAME_RELEASE" | sed -e 's/[-_].*//' | cut -d. -f1,2` + ;; + esac + # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: + # contains redundant information, the shorter form: + # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. + echo "$machine-${os}${release}${abi-}" + exit ;; + *:Bitrig:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` + echo "$UNAME_MACHINE_ARCH"-unknown-bitrig"$UNAME_RELEASE" + exit ;; + *:OpenBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo "$UNAME_MACHINE_ARCH"-unknown-openbsd"$UNAME_RELEASE" + exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo "$UNAME_MACHINE_ARCH"-unknown-libertybsd"$UNAME_RELEASE" + exit ;; + *:MidnightBSD:*:*) + echo "$UNAME_MACHINE"-unknown-midnightbsd"$UNAME_RELEASE" + exit ;; + *:ekkoBSD:*:*) + echo "$UNAME_MACHINE"-unknown-ekkobsd"$UNAME_RELEASE" + exit ;; + *:SolidBSD:*:*) + echo "$UNAME_MACHINE"-unknown-solidbsd"$UNAME_RELEASE" + exit ;; + macppc:MirBSD:*:*) + echo powerpc-unknown-mirbsd"$UNAME_RELEASE" + exit ;; + *:MirBSD:*:*) + echo "$UNAME_MACHINE"-unknown-mirbsd"$UNAME_RELEASE" + exit ;; + *:Sortix:*:*) + echo "$UNAME_MACHINE"-unknown-sortix + exit ;; + *:Redox:*:*) + echo "$UNAME_MACHINE"-unknown-redox + exit ;; + mips:OSF1:*.*) + echo mips-dec-osf1 + exit ;; + alpha:OSF1:*:*) + case $UNAME_RELEASE in + *4.0) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + ;; + *5.*) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + ;; + esac + # According to Compaq, /usr/sbin/psrinfo has been available on + # OSF/1 and Tru64 systems produced since 1995. I hope that + # covers most systems running today. This code pipes the CPU + # types through head -n 1, so we only detect the type of CPU 0. + ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + case "$ALPHA_CPU_TYPE" in + "EV4 (21064)") + UNAME_MACHINE=alpha ;; + "EV4.5 (21064)") + UNAME_MACHINE=alpha ;; + "LCA4 (21066/21068)") + UNAME_MACHINE=alpha ;; + "EV5 (21164)") + UNAME_MACHINE=alphaev5 ;; + "EV5.6 (21164A)") + UNAME_MACHINE=alphaev56 ;; + "EV5.6 (21164PC)") + UNAME_MACHINE=alphapca56 ;; + "EV5.7 (21164PC)") + UNAME_MACHINE=alphapca57 ;; + "EV6 (21264)") + UNAME_MACHINE=alphaev6 ;; + "EV6.7 (21264A)") + UNAME_MACHINE=alphaev67 ;; + "EV6.8CB (21264C)") + UNAME_MACHINE=alphaev68 ;; + "EV6.8AL (21264B)") + UNAME_MACHINE=alphaev68 ;; + "EV6.8CX (21264D)") + UNAME_MACHINE=alphaev68 ;; + "EV6.9A (21264/EV69A)") + UNAME_MACHINE=alphaev69 ;; + "EV7 (21364)") + UNAME_MACHINE=alphaev7 ;; + "EV7.9 (21364A)") + UNAME_MACHINE=alphaev79 ;; + esac + # A Pn.n version is a patched version. + # A Vn.n version is a released version. + # A Tn.n version is a released field test version. + # A Xn.n version is an unreleased experimental baselevel. + # 1.2 uses "1.2" for uname -r. + echo "$UNAME_MACHINE"-dec-osf"`echo "$UNAME_RELEASE" | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz`" + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; + Amiga*:UNIX_System_V:4.0:*) + echo m68k-unknown-sysv4 + exit ;; + *:[Aa]miga[Oo][Ss]:*:*) + echo "$UNAME_MACHINE"-unknown-amigaos + exit ;; + *:[Mm]orph[Oo][Ss]:*:*) + echo "$UNAME_MACHINE"-unknown-morphos + exit ;; + *:OS/390:*:*) + echo i370-ibm-openedition + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; + *:OS400:*:*) + echo powerpc-ibm-os400 + exit ;; + arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) + echo arm-acorn-riscix"$UNAME_RELEASE" + exit ;; + arm*:riscos:*:*|arm*:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; + SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) + echo hppa1.1-hitachi-hiuxmpp + exit ;; + Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) + # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. + if test "`(/bin/universe) 2>/dev/null`" = att ; then + echo pyramid-pyramid-sysv3 + else + echo pyramid-pyramid-bsd + fi + exit ;; + NILE*:*:*:dcosx) + echo pyramid-pyramid-svr4 + exit ;; + DRS?6000:unix:4.0:6*) + echo sparc-icl-nx6 + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) + case `/usr/bin/uname -p` in + sparc) echo sparc-icl-nx7; exit ;; + esac ;; + s390x:SunOS:*:*) + echo "$UNAME_MACHINE"-ibm-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + exit ;; + sun4H:SunOS:5.*:*) + echo sparc-hal-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + exit ;; + sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) + echo sparc-sun-solaris2"`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux"$UNAME_RELEASE" + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + UNAME_REL="`echo "$UNAME_RELEASE" | sed -e 's/[^.]*//'`" + case `isainfo -b` in + 32) + echo i386-pc-solaris2"$UNAME_REL" + ;; + 64) + echo x86_64-pc-solaris2"$UNAME_REL" + ;; + esac + exit ;; + sun4*:SunOS:6*:*) + # According to config.sub, this is the proper way to canonicalize + # SunOS6. Hard to guess exactly what SunOS6 will be like, but + # it's likely to be more like Solaris than SunOS4. + echo sparc-sun-solaris3"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + exit ;; + sun4*:SunOS:*:*) + case "`/usr/bin/arch -k`" in + Series*|S4*) + UNAME_RELEASE=`uname -v` + ;; + esac + # Japanese Language versions have a version number like `4.1.3-JL'. + echo sparc-sun-sunos"`echo "$UNAME_RELEASE"|sed -e 's/-/_/'`" + exit ;; + sun3*:SunOS:*:*) + echo m68k-sun-sunos"$UNAME_RELEASE" + exit ;; + sun*:*:4.2BSD:*) + UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + test "x$UNAME_RELEASE" = x && UNAME_RELEASE=3 + case "`/bin/arch`" in + sun3) + echo m68k-sun-sunos"$UNAME_RELEASE" + ;; + sun4) + echo sparc-sun-sunos"$UNAME_RELEASE" + ;; + esac + exit ;; + aushp:SunOS:*:*) + echo sparc-auspex-sunos"$UNAME_RELEASE" + exit ;; + # The situation for MiNT is a little confusing. The machine name + # can be virtually everything (everything which is not + # "atarist" or "atariste" at least should have a processor + # > m68000). The system name ranges from "MiNT" over "FreeMiNT" + # to the lowercase version "mint" (or "freemint"). Finally + # the system name "TOS" denotes a system which is actually not + # MiNT. But MiNT is downward compatible to TOS, so this should + # be no problem. + atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint"$UNAME_RELEASE" + exit ;; + atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint"$UNAME_RELEASE" + exit ;; + *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) + echo m68k-atari-mint"$UNAME_RELEASE" + exit ;; + milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) + echo m68k-milan-mint"$UNAME_RELEASE" + exit ;; + hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) + echo m68k-hades-mint"$UNAME_RELEASE" + exit ;; + *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) + echo m68k-unknown-mint"$UNAME_RELEASE" + exit ;; + m68k:machten:*:*) + echo m68k-apple-machten"$UNAME_RELEASE" + exit ;; + powerpc:machten:*:*) + echo powerpc-apple-machten"$UNAME_RELEASE" + exit ;; + RISC*:Mach:*:*) + echo mips-dec-mach_bsd4.3 + exit ;; + RISC*:ULTRIX:*:*) + echo mips-dec-ultrix"$UNAME_RELEASE" + exit ;; + VAX*:ULTRIX*:*:*) + echo vax-dec-ultrix"$UNAME_RELEASE" + exit ;; + 2020:CLIX:*:* | 2430:CLIX:*:*) + echo clipper-intergraph-clix"$UNAME_RELEASE" + exit ;; + mips:*:*:UMIPS | mips:*:*:RISCos) + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" +#ifdef __cplusplus +#include /* for printf() prototype */ + int main (int argc, char *argv[]) { +#else + int main (argc, argv) int argc; char *argv[]; { +#endif + #if defined (host_mips) && defined (MIPSEB) + #if defined (SYSTYPE_SYSV) + printf ("mips-mips-riscos%ssysv\\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_SVR4) + printf ("mips-mips-riscos%ssvr4\\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) + printf ("mips-mips-riscos%sbsd\\n", argv[1]); exit (0); + #endif + #endif + exit (-1); + } +EOF + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && + dummyarg=`echo "$UNAME_RELEASE" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`"$dummy" "$dummyarg"` && + { echo "$SYSTEM_NAME"; exit; } + echo mips-mips-riscos"$UNAME_RELEASE" + exit ;; + Motorola:PowerMAX_OS:*:*) + echo powerpc-motorola-powermax + exit ;; + Motorola:*:4.3:PL8-*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) + echo powerpc-harris-powermax + exit ;; + Night_Hawk:Power_UNIX:*:*) + echo powerpc-harris-powerunix + exit ;; + m88k:CX/UX:7*:*) + echo m88k-harris-cxux7 + exit ;; + m88k:*:4*:R4*) + echo m88k-motorola-sysv4 + exit ;; + m88k:*:3*:R3*) + echo m88k-motorola-sysv3 + exit ;; + AViiON:dgux:*:*) + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` + if [ "$UNAME_PROCESSOR" = mc88100 ] || [ "$UNAME_PROCESSOR" = mc88110 ] + then + if [ "$TARGET_BINARY_INTERFACE"x = m88kdguxelfx ] || \ + [ "$TARGET_BINARY_INTERFACE"x = x ] + then + echo m88k-dg-dgux"$UNAME_RELEASE" + else + echo m88k-dg-dguxbcs"$UNAME_RELEASE" + fi + else + echo i586-dg-dgux"$UNAME_RELEASE" + fi + exit ;; + M88*:DolphinOS:*:*) # DolphinOS (SVR3) + echo m88k-dolphin-sysv3 + exit ;; + M88*:*:R3*:*) + # Delta 88k system running SVR3 + echo m88k-motorola-sysv3 + exit ;; + XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) + echo m88k-tektronix-sysv3 + exit ;; + Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) + echo m68k-tektronix-bsd + exit ;; + *:IRIX*:*:*) + echo mips-sgi-irix"`echo "$UNAME_RELEASE"|sed -e 's/-/_/g'`" + exit ;; + ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + i*86:AIX:*:*) + echo i386-ibm-aix + exit ;; + ia64:AIX:*:*) + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" + fi + echo "$UNAME_MACHINE"-ibm-aix"$IBM_REV" + exit ;; + *:AIX:2:3) + if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" + #include + + main() + { + if (!__power_pc()) + exit(1); + puts("powerpc-ibm-aix3.2.5"); + exit(0); + } +EOF + if $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi + elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then + echo rs6000-ibm-aix3.2.4 + else + echo rs6000-ibm-aix3.2 + fi + exit ;; + *:AIX:*:[4567]) + IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + if /usr/sbin/lsattr -El "$IBM_CPU_ID" | grep ' POWER' >/dev/null 2>&1; then + IBM_ARCH=rs6000 + else + IBM_ARCH=powerpc + fi + if [ -x /usr/bin/lslpp ] ; then + IBM_REV=`/usr/bin/lslpp -Lqc bos.rte.libc | + awk -F: '{ print $3 }' | sed s/[0-9]*$/0/` + else + IBM_REV="$UNAME_VERSION.$UNAME_RELEASE" + fi + echo "$IBM_ARCH"-ibm-aix"$IBM_REV" + exit ;; + *:AIX:*:*) + echo rs6000-ibm-aix + exit ;; + ibmrt:4.4BSD:*|romp-ibm:4.4BSD:*) + echo romp-ibm-bsd4.4 + exit ;; + ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and + echo romp-ibm-bsd"$UNAME_RELEASE" # 4.3 with uname added to + exit ;; # report: romp-ibm BSD 4.3 + *:BOSX:*:*) + echo rs6000-bull-bosx + exit ;; + DPX/2?00:B.O.S.:*:*) + echo m68k-bull-sysv3 + exit ;; + 9000/[34]??:4.3bsd:1.*:*) + echo m68k-hp-bsd + exit ;; + hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) + echo m68k-hp-bsd4.4 + exit ;; + 9000/[34678]??:HP-UX:*:*) + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + case "$UNAME_MACHINE" in + 9000/31?) HP_ARCH=m68000 ;; + 9000/[34]??) HP_ARCH=m68k ;; + 9000/[678][0-9][0-9]) + if [ -x /usr/bin/getconf ]; then + sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "$sc_cpu_version" in + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "$sc_kernel_bits" in + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 + esac ;; + esac + fi + if [ "$HP_ARCH" = "" ]; then + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" + + #define _HPUX_SOURCE + #include + #include + + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); + + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } +EOF + (CCOPTS="" $CC_FOR_BUILD -o "$dummy" "$dummy.c" 2>/dev/null) && HP_ARCH=`"$dummy"` + test -z "$HP_ARCH" && HP_ARCH=hppa + fi ;; + esac + if [ "$HP_ARCH" = hppa2.0w ] + then + set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | + grep -q __LP64__ + then + HP_ARCH=hppa2.0w + else + HP_ARCH=hppa64 + fi + fi + echo "$HP_ARCH"-hp-hpux"$HPUX_REV" + exit ;; + ia64:HP-UX:*:*) + HPUX_REV=`echo "$UNAME_RELEASE"|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux"$HPUX_REV" + exit ;; + 3050*:HI-UX:*:*) + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" + #include + int + main () + { + long cpu = sysconf (_SC_CPU_VERSION); + /* The order matters, because CPU_IS_HP_MC68K erroneously returns + true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct + results, however. */ + if (CPU_IS_PA_RISC (cpu)) + { + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; + case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; + default: puts ("hppa-hitachi-hiuxwe2"); break; + } + } + else if (CPU_IS_HP_MC68K (cpu)) + puts ("m68k-hitachi-hiuxwe2"); + else puts ("unknown-hitachi-hiuxwe2"); + exit (0); + } +EOF + $CC_FOR_BUILD -o "$dummy" "$dummy.c" && SYSTEM_NAME=`"$dummy"` && + { echo "$SYSTEM_NAME"; exit; } + echo unknown-hitachi-hiuxwe2 + exit ;; + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:*) + echo hppa1.1-hp-bsd + exit ;; + 9000/8??:4.3bsd:*:*) + echo hppa1.0-hp-bsd + exit ;; + *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) + echo hppa1.0-hp-mpeix + exit ;; + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:*) + echo hppa1.1-hp-osf + exit ;; + hp8??:OSF1:*:*) + echo hppa1.0-hp-osf + exit ;; + i*86:OSF1:*:*) + if [ -x /usr/sbin/sysversion ] ; then + echo "$UNAME_MACHINE"-unknown-osf1mk + else + echo "$UNAME_MACHINE"-unknown-osf1 + fi + exit ;; + parisc*:Lites*:*:*) + echo hppa1.1-hp-lites + exit ;; + C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) + echo c1-convex-bsd + exit ;; + C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit ;; + C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) + echo c34-convex-bsd + exit ;; + C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) + echo c38-convex-bsd + exit ;; + C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) + echo c4-convex-bsd + exit ;; + CRAY*Y-MP:*:*:*) + echo ymp-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*[A-Z]90:*:*:*) + echo "$UNAME_MACHINE"-cray-unicos"$UNAME_RELEASE" \ + | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ + -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ + -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*TS:*:*:*) + echo t90-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*T3E:*:*:*) + echo alphaev5-cray-unicosmk"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' + exit ;; + CRAY*SV1:*:*:*) + echo sv1-cray-unicos"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' + exit ;; + *:UNICOS/mp:*:*) + echo craynv-cray-unicosmp"$UNAME_RELEASE" | sed -e 's/\.[^.]*$/.X/' + exit ;; + F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + 5000:UNIX_System_V:4.*:*) + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo "$UNAME_RELEASE" | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; + i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) + echo "$UNAME_MACHINE"-pc-bsdi"$UNAME_RELEASE" + exit ;; + sparc*:BSD/OS:*:*) + echo sparc-unknown-bsdi"$UNAME_RELEASE" + exit ;; + *:BSD/OS:*:*) + echo "$UNAME_MACHINE"-unknown-bsdi"$UNAME_RELEASE" + exit ;; + arm*:FreeBSD:*:*) + UNAME_PROCESSOR=`uname -p` + set_cc_for_build + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabi + else + echo "${UNAME_PROCESSOR}"-unknown-freebsd"`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`"-gnueabihf + fi + exit ;; + *:FreeBSD:*:*) + UNAME_PROCESSOR=`/usr/bin/uname -p` + case "$UNAME_PROCESSOR" in + amd64) + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; + esac + echo "$UNAME_PROCESSOR"-unknown-freebsd"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" + exit ;; + i*:CYGWIN*:*) + echo "$UNAME_MACHINE"-pc-cygwin + exit ;; + *:MINGW64*:*) + echo "$UNAME_MACHINE"-pc-mingw64 + exit ;; + *:MINGW*:*) + echo "$UNAME_MACHINE"-pc-mingw32 + exit ;; + *:MSYS*:*) + echo "$UNAME_MACHINE"-pc-msys + exit ;; + i*:PW*:*) + echo "$UNAME_MACHINE"-pc-pw32 + exit ;; + *:Interix*:*) + case "$UNAME_MACHINE" in + x86) + echo i586-pc-interix"$UNAME_RELEASE" + exit ;; + authenticamd | genuineintel | EM64T) + echo x86_64-unknown-interix"$UNAME_RELEASE" + exit ;; + IA64) + echo ia64-unknown-interix"$UNAME_RELEASE" + exit ;; + esac ;; + i*:UWIN*:*) + echo "$UNAME_MACHINE"-pc-uwin + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; + prep*:SunOS:5.*:*) + echo powerpcle-unknown-solaris2"`echo "$UNAME_RELEASE"|sed -e 's/[^.]*//'`" + exit ;; + *:GNU:*:*) + # the GNU system + echo "`echo "$UNAME_MACHINE"|sed -e 's,[-/].*$,,'`-unknown-$LIBC`echo "$UNAME_RELEASE"|sed -e 's,/.*$,,'`" + exit ;; + *:GNU/*:*:*) + # other systems with GNU libc and userland + echo "$UNAME_MACHINE-unknown-`echo "$UNAME_SYSTEM" | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`-$LIBC" + exit ;; + *:Minix:*:*) + echo "$UNAME_MACHINE"-unknown-minix + exit ;; + aarch64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + arc:Linux:*:* | arceb:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + arm*:Linux:*:*) + set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + else + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabi + else + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC"eabihf + fi + fi + exit ;; + avr32*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + cris:Linux:*:*) + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" + exit ;; + crisv32:Linux:*:*) + echo "$UNAME_MACHINE"-axis-linux-"$LIBC" + exit ;; + e2k:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + frv:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + hexagon:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + i*86:Linux:*:*) + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" + exit ;; + ia64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + k1om:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + m32r*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + m68*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + mips:Linux:*:* | mips64:Linux:*:*) + set_cc_for_build + sed 's/^ //' << EOF > "$dummy.c" + #undef CPU + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=${UNAME_MACHINE}el + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=${UNAME_MACHINE} + #else + CPU= + #endif + #endif +EOF + eval "`$CC_FOR_BUILD -E "$dummy.c" 2>/dev/null | grep '^CPU'`" + test "x$CPU" != x && { echo "$CPU-unknown-linux-$LIBC"; exit; } + ;; + mips64el:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + openrisc*:Linux:*:*) + echo or1k-unknown-linux-"$LIBC" + exit ;; + or32:Linux:*:* | or1k*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + padre:Linux:*:*) + echo sparc-unknown-linux-"$LIBC" + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-"$LIBC" + exit ;; + parisc:Linux:*:* | hppa:Linux:*:*) + # Look for CPU level + case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + PA7*) echo hppa1.1-unknown-linux-"$LIBC" ;; + PA8*) echo hppa2.0-unknown-linux-"$LIBC" ;; + *) echo hppa-unknown-linux-"$LIBC" ;; + esac + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-"$LIBC" + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-"$LIBC" + exit ;; + ppc64le:Linux:*:*) + echo powerpc64le-unknown-linux-"$LIBC" + exit ;; + ppcle:Linux:*:*) + echo powerpcle-unknown-linux-"$LIBC" + exit ;; + riscv32:Linux:*:* | riscv64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + s390:Linux:*:* | s390x:Linux:*:*) + echo "$UNAME_MACHINE"-ibm-linux-"$LIBC" + exit ;; + sh64*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + sh*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + sparc:Linux:*:* | sparc64:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + tile*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + vax:Linux:*:*) + echo "$UNAME_MACHINE"-dec-linux-"$LIBC" + exit ;; + x86_64:Linux:*:*) + echo "$UNAME_MACHINE"-pc-linux-"$LIBC" + exit ;; + xtensa*:Linux:*:*) + echo "$UNAME_MACHINE"-unknown-linux-"$LIBC" + exit ;; + i*86:DYNIX/ptx:4*:*) + # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. + # earlier versions are messed up and put the nodename in both + # sysname and nodename. + echo i386-sequent-sysv4 + exit ;; + i*86:UNIX_SV:4.2MP:2.*) + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, + # I just have to hope. -- rms. + # Use sysv4.2uw... so that sysv4* matches it. + echo "$UNAME_MACHINE"-pc-sysv4.2uw"$UNAME_VERSION" + exit ;; + i*86:OS/2:*:*) + # If we were able to find `uname', then EMX Unix compatibility + # is probably installed. + echo "$UNAME_MACHINE"-pc-os2-emx + exit ;; + i*86:XTS-300:*:STOP) + echo "$UNAME_MACHINE"-unknown-stop + exit ;; + i*86:atheos:*:*) + echo "$UNAME_MACHINE"-unknown-atheos + exit ;; + i*86:syllable:*:*) + echo "$UNAME_MACHINE"-pc-syllable + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) + echo i386-unknown-lynxos"$UNAME_RELEASE" + exit ;; + i*86:*DOS:*:*) + echo "$UNAME_MACHINE"-pc-msdosdjgpp + exit ;; + i*86:*:4.*:*) + UNAME_REL=`echo "$UNAME_RELEASE" | sed 's/\/MP$//'` + if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then + echo "$UNAME_MACHINE"-univel-sysv"$UNAME_REL" + else + echo "$UNAME_MACHINE"-pc-sysv"$UNAME_REL" + fi + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. + case `/bin/uname -X | grep "^Machine"` in + *486*) UNAME_MACHINE=i486 ;; + *Pentium) UNAME_MACHINE=i586 ;; + *Pent*|*Celeron) UNAME_MACHINE=i686 ;; + esac + echo "$UNAME_MACHINE-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}{$UNAME_VERSION}" + exit ;; + i*86:*:3.2:*) + if test -f /usr/options/cb.name; then + UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then + UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 + (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ + && UNAME_MACHINE=i586 + (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ + && UNAME_MACHINE=i686 + (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ + && UNAME_MACHINE=i686 + echo "$UNAME_MACHINE"-pc-sco"$UNAME_REL" + else + echo "$UNAME_MACHINE"-pc-sysv32 + fi + exit ;; + pc:*:*:*) + # Left here for compatibility: + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i586. + # Note: whatever this is, it MUST be the same as what config.sub + # prints for the "djgpp" host, or else GDB configure will decide that + # this is a cross-build. + echo i586-pc-msdosdjgpp + exit ;; + Intel:Mach:3*:*) + echo i386-pc-mach3 + exit ;; + paragon:*:*:*) + echo i860-intel-osf1 + exit ;; + i860:*:4.*:*) # i860-SVR4 + if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then + echo i860-stardent-sysv"$UNAME_RELEASE" # Stardent Vistra i860-SVR4 + else # Add other i860-SVR4 vendors below as they are discovered. + echo i860-unknown-sysv"$UNAME_RELEASE" # Unknown i860-SVR4 + fi + exit ;; + mini*:CTIX:SYS*5:*) + # "miniframe" + echo m68010-convergent-sysv + exit ;; + mc68k:UNIX:SYSTEM5:3.51m) + echo m68k-convergent-sysv + exit ;; + M680?0:D-NIX:5.3:*) + echo m68k-diab-dnix + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) + OS_REL='' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; + 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + NCR*:*:4.2:* | MPRAS*:*:4.2:*) + OS_REL='.3' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3"$OS_REL"; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } + /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ + && { echo i586-ncr-sysv4.3"$OS_REL"; exit; } ;; + m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) + echo m68k-unknown-lynxos"$UNAME_RELEASE" + exit ;; + mc68030:UNIX_System_V:4.*:*) + echo m68k-atari-sysv4 + exit ;; + TSUNAMI:LynxOS:2.*:*) + echo sparc-unknown-lynxos"$UNAME_RELEASE" + exit ;; + rs6000:LynxOS:2.*:*) + echo rs6000-unknown-lynxos"$UNAME_RELEASE" + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) + echo powerpc-unknown-lynxos"$UNAME_RELEASE" + exit ;; + SM[BE]S:UNIX_SV:*:*) + echo mips-dde-sysv"$UNAME_RELEASE" + exit ;; + RM*:ReliantUNIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + RM*:SINIX-*:*:*) + echo mips-sni-sysv4 + exit ;; + *:SINIX-*:*:*) + if uname -p 2>/dev/null >/dev/null ; then + UNAME_MACHINE=`(uname -p) 2>/dev/null` + echo "$UNAME_MACHINE"-sni-sysv4 + else + echo ns32k-sni-sysv + fi + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit ;; + *:UNIX_System_V:4*:FTX*) + # From Gerald Hewes . + # How about differentiating between stratus architectures? -djm + echo hppa1.1-stratus-sysv4 + exit ;; + *:*:*:FTX*) + # From seanf@swdc.stratus.com. + echo i860-stratus-sysv4 + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo "$UNAME_MACHINE"-stratus-vos + exit ;; + *:VOS:*:*) + # From Paul.Green@stratus.com. + echo hppa1.1-stratus-vos + exit ;; + mc68*:A/UX:*:*) + echo m68k-apple-aux"$UNAME_RELEASE" + exit ;; + news*:NEWS-OS:6*:*) + echo mips-sony-newsos6 + exit ;; + R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) + if [ -d /usr/nec ]; then + echo mips-nec-sysv"$UNAME_RELEASE" + else + echo mips-unknown-sysv"$UNAME_RELEASE" + fi + exit ;; + BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. + echo powerpc-be-beos + exit ;; + BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. + echo powerpc-apple-beos + exit ;; + BePC:BeOS:*:*) # BeOS running on Intel PC compatible. + echo i586-pc-beos + exit ;; + BePC:Haiku:*:*) # Haiku running on Intel PC compatible. + echo i586-pc-haiku + exit ;; + x86_64:Haiku:*:*) + echo x86_64-unknown-haiku + exit ;; + SX-4:SUPER-UX:*:*) + echo sx4-nec-superux"$UNAME_RELEASE" + exit ;; + SX-5:SUPER-UX:*:*) + echo sx5-nec-superux"$UNAME_RELEASE" + exit ;; + SX-6:SUPER-UX:*:*) + echo sx6-nec-superux"$UNAME_RELEASE" + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux"$UNAME_RELEASE" + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux"$UNAME_RELEASE" + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux"$UNAME_RELEASE" + exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux"$UNAME_RELEASE" + exit ;; + Power*:Rhapsody:*:*) + echo powerpc-apple-rhapsody"$UNAME_RELEASE" + exit ;; + *:Rhapsody:*:*) + echo "$UNAME_MACHINE"-apple-rhapsody"$UNAME_RELEASE" + exit ;; + *:Darwin:*:*) + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + set_cc_for_build + if test "$UNAME_PROCESSOR" = unknown ; then + UNAME_PROCESSOR=powerpc + fi + if test "`echo "$UNAME_RELEASE" | sed -e 's/\..*//'`" -le 10 ; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + case $UNAME_PROCESSOR in + i386) UNAME_PROCESSOR=x86_64 ;; + powerpc) UNAME_PROCESSOR=powerpc64 ;; + esac + fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi + fi + elif test "$UNAME_PROCESSOR" = i386 ; then + # Avoid executing cc on OS X 10.9, as it ships with a stub + # that puts up a graphical alert prompting to install + # developer tools. Any system running Mac OS X 10.7 or + # later (Darwin 11 and later) is required to have a 64-bit + # processor. This is not true of the ARM version of Darwin + # that Apple uses in portable devices. + UNAME_PROCESSOR=x86_64 + fi + echo "$UNAME_PROCESSOR"-apple-darwin"$UNAME_RELEASE" + exit ;; + *:procnto*:*:* | *:QNX:[0123456789]*:*) + UNAME_PROCESSOR=`uname -p` + if test "$UNAME_PROCESSOR" = x86; then + UNAME_PROCESSOR=i386 + UNAME_MACHINE=pc + fi + echo "$UNAME_PROCESSOR"-"$UNAME_MACHINE"-nto-qnx"$UNAME_RELEASE" + exit ;; + *:QNX:*:4*) + echo i386-pc-qnx + exit ;; + NEO-*:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSE-*:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSR-*:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSV-*:NONSTOP_KERNEL:*:*) + echo nsv-tandem-nsk"$UNAME_RELEASE" + exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk"$UNAME_RELEASE" + exit ;; + *:NonStop-UX:*:*) + echo mips-compaq-nonstopux + exit ;; + BS2000:POSIX*:*:*) + echo bs2000-siemens-sysv + exit ;; + DS/*:UNIX_System_V:*:*) + echo "$UNAME_MACHINE"-"$UNAME_SYSTEM"-"$UNAME_RELEASE" + exit ;; + *:Plan9:*:*) + # "uname -m" is not consistent, so use $cputype instead. 386 + # is converted to i386 for consistency with other x86 + # operating systems. + # shellcheck disable=SC2154 + if test "$cputype" = 386; then + UNAME_MACHINE=i386 + else + UNAME_MACHINE="$cputype" + fi + echo "$UNAME_MACHINE"-unknown-plan9 + exit ;; + *:TOPS-10:*:*) + echo pdp10-unknown-tops10 + exit ;; + *:TENEX:*:*) + echo pdp10-unknown-tenex + exit ;; + KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) + echo pdp10-dec-tops20 + exit ;; + XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) + echo pdp10-xkl-tops20 + exit ;; + *:TOPS-20:*:*) + echo pdp10-unknown-tops20 + exit ;; + *:ITS:*:*) + echo pdp10-unknown-its + exit ;; + SEI:*:*:SEIUX) + echo mips-sei-seiux"$UNAME_RELEASE" + exit ;; + *:DragonFly:*:*) + echo "$UNAME_MACHINE"-unknown-dragonfly"`echo "$UNAME_RELEASE"|sed -e 's/[-(].*//'`" + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "$UNAME_MACHINE" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo "$UNAME_MACHINE"-pc-skyos"`echo "$UNAME_RELEASE" | sed -e 's/ .*$//'`" + exit ;; + i*86:rdos:*:*) + echo "$UNAME_MACHINE"-pc-rdos + exit ;; + i*86:AROS:*:*) + echo "$UNAME_MACHINE"-pc-aros + exit ;; + x86_64:VMkernel:*:*) + echo "$UNAME_MACHINE"-unknown-esx + exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-unknown-onefs + exit ;; +esac + +echo "$0: unable to guess system type" >&2 + +case "$UNAME_MACHINE:$UNAME_SYSTEM" in + mips:Linux | mips64:Linux) + # If we got here on MIPS GNU/Linux, output extra information. + cat >&2 <&2 </dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null` + +hostinfo = `(hostinfo) 2>/dev/null` +/bin/universe = `(/bin/universe) 2>/dev/null` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` +/bin/arch = `(/bin/arch) 2>/dev/null` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` + +UNAME_MACHINE = "$UNAME_MACHINE" +UNAME_RELEASE = "$UNAME_RELEASE" +UNAME_SYSTEM = "$UNAME_SYSTEM" +UNAME_VERSION = "$UNAME_VERSION" +EOF + +exit 1 + +# Local variables: +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/third_party/nix/config/config.sub b/third_party/nix/config/config.sub new file mode 100755 index 000000000..c19e67180 --- /dev/null +++ b/third_party/nix/config/config.sub @@ -0,0 +1,1818 @@ +#! /bin/sh +# Configuration validation subroutine script. +# Copyright 1992-2018 Free Software Foundation, Inc. + +timestamp='2018-08-13' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see . +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that +# program. This Exception is an additional permission under section 7 +# of the GNU General Public License, version 3 ("GPLv3"). + + +# Please send patches to . +# +# Configuration subroutine to validate and canonicalize a configuration type. +# Supply the specified configuration type as an argument. +# If it is invalid, we print an error message on stderr and exit with code 1. +# Otherwise, we print the canonical config type on stdout and succeed. + +# You can get the latest version of this script from: +# https://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub + +# This file is supposed to be the same for all GNU packages +# and recognize all the CPU types, system types and aliases +# that are meaningful with *any* GNU software. +# Each package is responsible for reporting which valid configurations +# it does not support. The user should be able to distinguish +# a failure to support a valid configuration from a meaningless +# configuration. + +# The goal of this file is to map all the various variations of a given +# machine specification into a single specification in the form: +# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM +# or in some cases, the newer four-part form: +# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM +# It is wrong to echo any other type of specification. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS + +Canonicalize a configuration name. + +Options: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.sub ($timestamp) + +Copyright 1992-2018 Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit ;; + --version | -v ) + echo "$version" ; exit ;; + --help | --h* | -h ) + echo "$usage"; exit ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" + exit 1 ;; + + *local*) + # First pass through any local machine types. + echo "$1" + exit ;; + + * ) + break ;; + esac +done + +case $# in + 0) echo "$me: missing argument$help" >&2 + exit 1;; + 1) ;; + *) echo "$me: too many arguments$help" >&2 + exit 1;; +esac + +# Split fields of configuration type +IFS="-" read -r field1 field2 field3 field4 <&2 + exit 1 + ;; + *-*-*-*) + basic_machine=$field1-$field2 + os=$field3-$field4 + ;; + *-*-*) + # Ambiguous whether COMPANY is present, or skipped and KERNEL-OS is two + # parts + maybe_os=$field2-$field3 + case $maybe_os in + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc \ + | linux-newlib* | linux-musl* | linux-uclibc* | uclinux-uclibc* \ + | uclinux-gnu* | kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* \ + | netbsd*-eabi* | kopensolaris*-gnu* | cloudabi*-eabi* \ + | storm-chaos* | os2-emx* | rtmk-nova*) + basic_machine=$field1 + os=$maybe_os + ;; + android-linux) + basic_machine=$field1-unknown + os=linux-android + ;; + *) + basic_machine=$field1-$field2 + os=$field3 + ;; + esac + ;; + *-*) + # Second component is usually, but not always the OS + case $field2 in + # Prevent following clause from handling this valid os + sun*os*) + basic_machine=$field1 + os=$field2 + ;; + # Manufacturers + dec* | mips* | sequent* | encore* | pc532* | sgi* | sony* \ + | att* | 7300* | 3300* | delta* | motorola* | sun[234]* \ + | unicom* | ibm* | next | hp | isi* | apollo | altos* \ + | convergent* | ncr* | news | 32* | 3600* | 3100* | hitachi* \ + | c[123]* | convex* | sun | crds | omron* | dg | ultra | tti* \ + | harris | dolphin | highlevel | gould | cbm | ns | masscomp \ + | apple | axis | knuth | cray | microblaze* \ + | sim | cisco | oki | wec | wrs | winbond) + basic_machine=$field1-$field2 + os= + ;; + *) + basic_machine=$field1 + os=$field2 + ;; + esac + ;; + *) + # Convert single-component short-hands not valid as part of + # multi-component configurations. + case $field1 in + 386bsd) + basic_machine=i386-pc + os=bsd + ;; + a29khif) + basic_machine=a29k-amd + os=udi + ;; + adobe68k) + basic_machine=m68010-adobe + os=scout + ;; + alliant) + basic_machine=fx80-alliant + os= + ;; + altos | altos3068) + basic_machine=m68k-altos + os= + ;; + am29k) + basic_machine=a29k-none + os=bsd + ;; + amdahl) + basic_machine=580-amdahl + os=sysv + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=bsd + ;; + aros) + basic_machine=i386-pc + os=aros + ;; + aux) + basic_machine=m68k-apple + os=aux + ;; + balance) + basic_machine=ns32k-sequent + os=dynix + ;; + blackfin) + basic_machine=bfin-unknown + os=linux + ;; + cegcc) + basic_machine=arm-unknown + os=cegcc + ;; + convex-c1) + basic_machine=c1-convex + os=bsd + ;; + convex-c2) + basic_machine=c2-convex + os=bsd + ;; + convex-c32) + basic_machine=c32-convex + os=bsd + ;; + convex-c34) + basic_machine=c34-convex + os=bsd + ;; + convex-c38) + basic_machine=c38-convex + os=bsd + ;; + cray) + basic_machine=j90-cray + os=unicos + ;; + crds | unos) + basic_machine=m68k-crds + os= + ;; + delta88) + basic_machine=m88k-motorola + os=sysv3 + ;; + dicos) + basic_machine=i686-pc + os=dicos + ;; + djgpp) + basic_machine=i586-pc + os=msdosdjgpp + ;; + ebmon29k) + basic_machine=a29k-amd + os=ebmon + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=ose + ;; + gmicro) + basic_machine=tron-gmicro + os=sysv + ;; + go32) + basic_machine=i386-pc + os=go32 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=hms + ;; + harris) + basic_machine=m88k-harris + os=sysv3 + ;; + hp300bsd) + basic_machine=m68k-hp + os=bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=hpux + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=proelf + ;; + i386mach) + basic_machine=i386-mach + os=mach + ;; + vsta) + basic_machine=i386-pc + os=vsta + ;; + isi68 | isi) + basic_machine=m68k-isi + os=sysv + ;; + m68knommu) + basic_machine=m68k-unknown + os=linux + ;; + magnum | m3230) + basic_machine=mips-mips + os=sysv + ;; + merlin) + basic_machine=ns32k-utek + os=sysv + ;; + mingw64) + basic_machine=x86_64-pc + os=mingw64 + ;; + mingw32) + basic_machine=i686-pc + os=mingw32 + ;; + mingw32ce) + basic_machine=arm-unknown + os=mingw32ce + ;; + monitor) + basic_machine=m68k-rom68k + os=coff + ;; + morphos) + basic_machine=powerpc-unknown + os=morphos + ;; + moxiebox) + basic_machine=moxie-unknown + os=moxiebox + ;; + msdos) + basic_machine=i386-pc + os=msdos + ;; + msys) + basic_machine=i686-pc + os=msys + ;; + mvs) + basic_machine=i370-ibm + os=mvs + ;; + nacl) + basic_machine=le32-unknown + os=nacl + ;; + ncr3000) + basic_machine=i486-ncr + os=sysv4 + ;; + netbsd386) + basic_machine=i386-pc + os=netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=newsos + ;; + news1000) + basic_machine=m68030-sony + os=newsos + ;; + necv70) + basic_machine=v70-nec + os=sysv + ;; + nh3000) + basic_machine=m68k-harris + os=cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=cxux + ;; + nindy960) + basic_machine=i960-intel + os=nindy + ;; + mon960) + basic_machine=i960-intel + os=mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=nonstopux + ;; + os400) + basic_machine=powerpc-ibm + os=os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=ose + ;; + os68k) + basic_machine=m68k-none + os=os68k + ;; + paragon) + basic_machine=i860-intel + os=osf + ;; + parisc) + basic_machine=hppa-unknown + os=linux + ;; + pw32) + basic_machine=i586-unknown + os=pw32 + ;; + rdos | rdos64) + basic_machine=x86_64-pc + os=rdos + ;; + rdos32) + basic_machine=i386-pc + os=rdos + ;; + rom68k) + basic_machine=m68k-rom68k + os=coff + ;; + sa29200) + basic_machine=a29k-amd + os=udi + ;; + sei) + basic_machine=mips-sei + os=seiux + ;; + sps7) + basic_machine=m68k-bull + os=sysv2 + ;; + st2000) + basic_machine=m68k-tandem + os= + ;; + stratus) + basic_machine=i860-stratus + os=sysv4 + ;; + sun2) + basic_machine=m68000-sun + os= + ;; + sun2os3) + basic_machine=m68000-sun + os=sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=sunos4 + ;; + sun3) + basic_machine=m68k-sun + os= + ;; + sun3os3) + basic_machine=m68k-sun + os=sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=sunos4 + ;; + sun4) + basic_machine=sparc-sun + os= + ;; + sun4os3) + basic_machine=sparc-sun + os=sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=solaris2 + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + os= + ;; + sv1) + basic_machine=sv1-cray + os=unicos + ;; + symmetry) + basic_machine=i386-sequent + os=dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=unicos + ;; + t90) + basic_machine=t90-cray + os=unicos + ;; + toad1) + basic_machine=pdp10-xkl + os=tops20 + ;; + tpf) + basic_machine=s390x-ibm + os=tpf + ;; + udi29k) + basic_machine=a29k-amd + os=udi + ;; + ultra3) + basic_machine=a29k-nyu + os=sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=none + ;; + vaxv) + basic_machine=vax-dec + os=sysv + ;; + vms) + basic_machine=vax-dec + os=vms + ;; + vxworks960) + basic_machine=i960-wrs + os=vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=vxworks + ;; + xbox) + basic_machine=i686-pc + os=mingw32 + ;; + ymp) + basic_machine=ymp-cray + os=unicos + ;; + *) + basic_machine=$1 + os= + ;; + esac + ;; +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Here we handle the default manufacturer of certain CPU types. It is in + # some cases the only manufacturer, in others, it is the most popular. + craynv) + basic_machine=craynv-cray + os=${os:-unicosmp} + ;; + fx80) + basic_machine=fx80-alliant + ;; + w89k) + basic_machine=hppa1.1-winbond + ;; + op50n) + basic_machine=hppa1.1-oki + ;; + op60c) + basic_machine=hppa1.1-oki + ;; + romp) + basic_machine=romp-ibm + ;; + mmix) + basic_machine=mmix-knuth + ;; + rs6000) + basic_machine=rs6000-ibm + ;; + vax) + basic_machine=vax-dec + ;; + pdp11) + basic_machine=pdp11-dec + ;; + we32k) + basic_machine=we32k-att + ;; + cydra) + basic_machine=cydra-cydrome + ;; + i370-ibm* | ibm*) + basic_machine=i370-ibm + ;; + orion) + basic_machine=orion-highlevel + ;; + orion105) + basic_machine=clipper-highlevel + ;; + mac | mpw | mac-mpw) + basic_machine=m68k-apple + ;; + pmac | pmac-mpw) + basic_machine=powerpc-apple + ;; + xps | xps100) + basic_machine=xps100-honeywell + ;; + + # Recognize the basic CPU types without company name. + # Some are omitted here because they have special meanings below. + 1750a | 580 \ + | a29k \ + | aarch64 | aarch64_be \ + | abacus \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ + | am33_2.0 \ + | arc | arceb \ + | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv6m | armv[78][arm] \ + | avr | avr32 \ + | asmjs \ + | ba \ + | be32 | be64 \ + | bfin \ + | c4x | c8051 | clipper | csky \ + | d10v | d30v | dlx | dsp16xx \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ + | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ + | i370 | i860 | i960 | ia16 | ia64 \ + | ip2k | iq2000 \ + | k1om \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | nvptx | picochip \ + | maxq | mb | microblaze | microblazeel | mcore | mep | metag \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64el \ + | mips64octeon | mips64octeonel \ + | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa32r6 | mipsisa32r6el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64r6 | mipsisa64r6el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipsr5900 | mipsr5900el \ + | mipstx39 | mipstx39el \ + | mn10200 | mn10300 \ + | moxie \ + | mt \ + | msp430 \ + | nds32 | nds32le | nds32be \ + | nfp \ + | nios | nios2 | nios2eb | nios2el \ + | ns16k | ns32k \ + | open8 | or1k | or1knd | or32 \ + | pdp10 | pj | pjl \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ + | pyramid \ + | riscv | riscv32 | riscv64 \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh[23]ele \ + | sh64 | sh64le \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ + | visium \ + | wasm32 \ + | x86 | xc16x | xstormy16 | xgate | xtensa \ + | z8k | z80) + basic_machine=$basic_machine-unknown + ;; + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + leon|leon[3-9]) + basic_machine=sparc-$basic_machine + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65) + ;; + m9s12z | m68hcs12z | hcs12z | s12z) + basic_machine=s12z-unknown + ;; + m9s12z-* | m68hcs12z-* | hcs12z-* | s12z-*) + basic_machine=s12z-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ms1) + basic_machine=mt-unknown + ;; + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; + + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + basic_machine=$basic_machine-pc + ;; + # Recognize the basic CPU types with company name. + 1750a-* | 580-* \ + | a29k-* \ + | aarch64-* | aarch64_be-* \ + | abacus-* \ + | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ + | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ + | alphapca5[67]-* | alpha64pca5[67]-* \ + | am33_2.0-* \ + | arc-* | arceb-* \ + | arm-* | arm[lb]e-* | arme[lb]-* | armv*-* \ + | avr-* | avr32-* \ + | asmjs-* \ + | ba-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | c8051-* | clipper-* | craynv-* | csky-* | cydra-* \ + | d10v-* | d30v-* | dlx-* | dsp16xx-* \ + | e2k-* | elxsi-* | epiphany-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | ft32-* | fx80-* \ + | h8300-* | h8500-* \ + | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ + | i370-* | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ + | ip2k-* | iq2000-* \ + | k1om-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ + | m5200-* | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* | v70-* | w65-* \ + | m6811-* | m68hc11-* | m6812-* | m68hc12-* | m68hcs12x-* | nvptx-* | picochip-* \ + | m88110-* | m88k-* | maxq-* | mb-* | mcore-* | mep-* | metag-* \ + | microblaze-* | microblazeel-* \ + | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ + | mips16-* \ + | mips64-* | mips64el-* \ + | mips64octeon-* | mips64octeonel-* \ + | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ + | mips64vr4100-* | mips64vr4100el-* \ + | mips64vr4300-* | mips64vr4300el-* \ + | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ + | mipsisa32-* | mipsisa32el-* \ + | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa32r6-* | mipsisa32r6el-* \ + | mipsisa64-* | mipsisa64el-* \ + | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64r6-* | mipsisa64r6el-* \ + | mipsisa64sb1-* | mipsisa64sb1el-* \ + | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipsr5900-* | mipsr5900el-* \ + | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mn10200-* | mn10300-* \ + | moxie-* \ + | mt-* \ + | msp430-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nfp-* \ + | nios-* | nios2-* | nios2eb-* | nios2el-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ + | or1k*-* \ + | or32-* \ + | orion-* \ + | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ + | pyramid-* \ + | riscv-* | riscv32-* | riscv64-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | score-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]ae[lb]-* | sh[23]e-* | she[lb]-* | sh[lb]e-* \ + | sh[1234]e[lb]-* | sh[12345][lb]e-* | sh[23]ele-* | sh64-* | sh64le-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ + | spu-* \ + | tahoe-* \ + | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tron-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ + | visium-* \ + | wasm32-* \ + | we32k-* \ + | x86-* | x86_64-* | xc16x-* | xgate-* | xps100-* \ + | xstormy16-* | xtensa*-* \ + | ymp-* \ + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown + ;; + # Recognize the various machine names and aliases which stand + # for a CPU type and a company and sometimes even an OS. + 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) + basic_machine=m68000-att + ;; + 3b*) + basic_machine=we32k-att + ;; + amd64) + basic_machine=x86_64-pc + ;; + amd64-*) + basic_machine=x86_64-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + amiga | amiga-*) + basic_machine=m68k-unknown + ;; + blackfin-*) + basic_machine=bfin-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=cnk + ;; + c54x-*) + basic_machine=tic54x-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + c90) + basic_machine=c90-cray + os=${os:-unicos} + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown + os=${os:-elf} + ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; + cris | cris-* | etrax*) + basic_machine=cris-axis + ;; + crx) + basic_machine=crx-unknown + os=${os:-elf} + ;; + da30 | da30-*) + basic_machine=m68k-da30 + ;; + decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) + basic_machine=mips-dec + ;; + decsystem10* | dec10*) + basic_machine=pdp10-dec + os=tops10 + ;; + decsystem20* | dec20*) + basic_machine=pdp10-dec + os=tops20 + ;; + delta | 3300 | motorola-3300 | motorola-delta \ + | 3300-motorola | delta-motorola) + basic_machine=m68k-motorola + ;; + dpx20 | dpx20-*) + basic_machine=rs6000-bull + os=${os:-bosx} + ;; + dpx2*) + basic_machine=m68k-bull + os=sysv3 + ;; + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" + ;; + e500v[12]-*) + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=$os"spe" + ;; + encore | umax | mmax) + basic_machine=ns32k-encore + ;; + elxsi) + basic_machine=elxsi-elxsi + os=${os:-bsd} + ;; + fx2800) + basic_machine=i860-alliant + ;; + genix) + basic_machine=ns32k-ns + ;; + h3050r* | hiux*) + basic_machine=hppa1.1-hitachi + os=hiuxwe2 + ;; + hp300-*) + basic_machine=m68k-hp + ;; + hp3k9[0-9][0-9] | hp9[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k2[0-9][0-9] | hp9k31[0-9]) + basic_machine=m68000-hp + ;; + hp9k3[2-9][0-9]) + basic_machine=m68k-hp + ;; + hp9k6[0-9][0-9] | hp6[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k7[0-79][0-9] | hp7[0-79][0-9]) + basic_machine=hppa1.1-hp + ;; + hp9k78[0-9] | hp78[0-9]) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][13679] | hp8[0-9][13679]) + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][0-9] | hp8[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + i*86v32) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv32 + ;; + i*86v4*) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv4 + ;; + i*86v) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=sysv + ;; + i*86sol2) + basic_machine=`echo "$1" | sed -e 's/86.*/86-pc/'` + os=solaris2 + ;; + j90 | j90-cray) + basic_machine=j90-cray + os=${os:-unicos} + ;; + iris | iris4d) + basic_machine=mips-sgi + case $os in + irix*) + ;; + *) + os=irix4 + ;; + esac + ;; + leon-*|leon[3-9]-*) + basic_machine=sparc-`echo "$basic_machine" | sed 's/-.*//'` + ;; + m68knommu-*) + basic_machine=m68k-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux + ;; + microblaze*) + basic_machine=microblaze-xilinx + ;; + miniframe) + basic_machine=m68000-convergent + ;; + *mint | mint[0-9]* | *MiNT | *MiNT[0-9]*) + basic_machine=m68k-atari + os=mint + ;; + mips3*-*) + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'` + ;; + mips3*) + basic_machine=`echo "$basic_machine" | sed -e 's/mips3/mips64/'`-unknown + ;; + ms1-*) + basic_machine=`echo "$basic_machine" | sed -e 's/ms1-/mt-/'` + ;; + news-3600 | risc-news) + basic_machine=mips-sony + os=newsos + ;; + next | m*-next) + basic_machine=m68k-next + case $os in + nextstep* ) + ;; + ns2*) + os=nextstep2 + ;; + *) + os=nextstep3 + ;; + esac + ;; + np1) + basic_machine=np1-gould + ;; + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem + ;; + nsr-tandem) + basic_machine=nsr-tandem + ;; + nsv-tandem) + basic_machine=nsv-tandem + ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; + op50n-* | op60c-*) + basic_machine=hppa1.1-oki + os=proelf + ;; + openrisc | openrisc-*) + basic_machine=or32-unknown + ;; + pa-hitachi) + basic_machine=hppa1.1-hitachi + os=hiuxwe2 + ;; + parisc-*) + basic_machine=hppa-`echo "$basic_machine" | sed 's/^[^-]*-//'` + os=linux + ;; + pbd) + basic_machine=sparc-tti + ;; + pbb) + basic_machine=m68k-tti + ;; + pc532 | pc532-*) + basic_machine=ns32k-pc532 + ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentium | p5 | k5 | k6 | nexgen | viac3) + basic_machine=i586-pc + ;; + pentiumpro | p6 | 6x86 | athlon | athlon_*) + basic_machine=i686-pc + ;; + pentiumii | pentium2 | pentiumiii | pentium3) + basic_machine=i686-pc + ;; + pentium4) + basic_machine=i786-pc + ;; + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + basic_machine=i586-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentiumpro-* | p6-* | 6x86-* | athlon-*) + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + basic_machine=i686-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pentium4-*) + basic_machine=i786-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + pn) + basic_machine=pn-gould + ;; + power) basic_machine=power-ibm + ;; + ppc | ppcbe) basic_machine=powerpc-unknown + ;; + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ppcle | powerpclittle) + basic_machine=powerpcle-unknown + ;; + ppcle-* | powerpclittle-*) + basic_machine=powerpcle-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ppc64) basic_machine=powerpc64-unknown + ;; + ppc64-*) basic_machine=powerpc64-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ppc64le | powerpc64little) + basic_machine=powerpc64le-unknown + ;; + ppc64le-* | powerpc64little-*) + basic_machine=powerpc64le-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + ps2) + basic_machine=i386-ibm + ;; + rm[46]00) + basic_machine=mips-siemens + ;; + rtpc | rtpc-*) + basic_machine=romp-ibm + ;; + s390 | s390-*) + basic_machine=s390-ibm + ;; + s390x | s390x-*) + basic_machine=s390x-ibm + ;; + sb1) + basic_machine=mipsisa64sb1-unknown + ;; + sb1el) + basic_machine=mipsisa64sb1el-unknown + ;; + sde) + basic_machine=mipsisa32-sde + os=${os:-elf} + ;; + sequent) + basic_machine=i386-sequent + ;; + sh5el) + basic_machine=sh5le-unknown + ;; + sh5el-*) + basic_machine=sh5le-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + simso-wrs) + basic_machine=sparclite-wrs + os=vxworks + ;; + spur) + basic_machine=spur-unknown + ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo "$basic_machine" | sed 's/^[^-]*-//'` + ;; + tile*-*) + ;; + tile*) + basic_machine=$basic_machine-unknown + os=${os:-linux-gnu} + ;; + tx39) + basic_machine=mipstx39-unknown + ;; + tx39el) + basic_machine=mipstx39el-unknown + ;; + tower | tower-32) + basic_machine=m68k-ncr + ;; + vpp*|vx|vx-*) + basic_machine=f301-fujitsu + ;; + w65*) + basic_machine=w65-wdc + os=none + ;; + w89k-*) + basic_machine=hppa1.1-winbond + os=proelf + ;; + x64) + basic_machine=x86_64-pc + ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo "$basic_machine" | sed 's/^xscale/arm/'` + ;; + none) + basic_machine=none-none + ;; + + *) + echo Invalid configuration \`"$1"\': machine \`"$basic_machine"\' not recognized 1>&2 + exit 1 + ;; +esac + +# Here we canonicalize certain aliases for manufacturers. +case $basic_machine in + *-digital*) + basic_machine=`echo "$basic_machine" | sed 's/digital.*/dec/'` + ;; + *-commodore*) + basic_machine=`echo "$basic_machine" | sed 's/commodore.*/cbm/'` + ;; + *) + ;; +esac + +# Decode manufacturer-specific aliases for certain operating systems. + +if [ x$os != x ] +then +case $os in + # First match some system type aliases that might get confused + # with valid system types. + # solaris* is a basic system type, with this one exception. + auroraux) + os=auroraux + ;; + bluegene*) + os=cnk + ;; + solaris1 | solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` + ;; + solaris) + os=solaris2 + ;; + unixware*) + os=sysv4.2uw + ;; + gnu/linux*) + os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + ;; + # es1800 is here to avoid being matched by es* (a different OS) + es1800*) + os=ose + ;; + # Some version numbers need modification + chorusos*) + os=chorusos + ;; + isc) + os=isc2.2 + ;; + sco6) + os=sco5v6 + ;; + sco5) + os=sco3.2v5 + ;; + sco4) + os=sco3.2v4 + ;; + sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + ;; + sco3.2v[4-9]* | sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + ;; + scout) + # Don't match below + ;; + sco*) + os=sco3.2v2 + ;; + psos*) + os=psos + ;; + # Now accept the basic system types. + # The portable systems comes first. + # Each alternative MUST end in a * to match a version number. + # sysv* is not here because it comes later, after sysvr4. + gnu* | bsd* | mach* | minix* | genix* | ultrix* | irix* \ + | *vms* | esix* | aix* | cnk* | sunos | sunos[34]*\ + | hpux* | unos* | osf* | luna* | dgux* | auroraux* | solaris* \ + | sym* | kopensolaris* | plan9* \ + | amigaos* | amigados* | msdos* | newsos* | unicos* | aof* \ + | aos* | aros* | cloudabi* | sortix* \ + | nindy* | vxsim* | vxworks* | ebmon* | hms* | mvs* \ + | clix* | riscos* | uniplus* | iris* | isc* | rtu* | xenix* \ + | knetbsd* | mirbsd* | netbsd* \ + | bitrig* | openbsd* | solidbsd* | libertybsd* \ + | ekkobsd* | kfreebsd* | freebsd* | riscix* | lynxos* \ + | bosx* | nextstep* | cxux* | aout* | elf* | oabi* \ + | ptx* | coff* | ecoff* | winnt* | domain* | vsta* \ + | udi* | eabi* | lites* | ieee* | go32* | aux* | hcos* \ + | chorusrdb* | cegcc* | glidix* \ + | cygwin* | msys* | pe* | moss* | proelf* | rtems* \ + | midipix* | mingw32* | mingw64* | linux-gnu* | linux-android* \ + | linux-newlib* | linux-musl* | linux-uclibc* \ + | uxpv* | beos* | mpeix* | udk* | moxiebox* \ + | interix* | uwin* | mks* | rhapsody* | darwin* \ + | openstep* | oskit* | conix* | pw32* | nonstopux* \ + | storm-chaos* | tops10* | tenex* | tops20* | its* \ + | os2* | vos* | palmos* | uclinux* | nucleus* \ + | morphos* | superux* | rtmk* | windiss* \ + | powermax* | dnix* | nx6 | nx7 | sei* | dragonfly* \ + | skyos* | haiku* | rdos* | toppers* | drops* | es* \ + | onefs* | tirtos* | phoenix* | fuchsia* | redox* | bme* \ + | midnightbsd*) + # Remember, each alternative MUST END IN *, to match a version number. + ;; + qnx*) + case $basic_machine in + x86-* | i*86-*) + ;; + *) + os=nto-$os + ;; + esac + ;; + hiux*) + os=hiuxwe2 + ;; + nto-qnx*) + ;; + nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` + ;; + sim | xray | os68k* | v88r* \ + | windows* | osx | abug | netware* | os9* \ + | macos* | mpw* | magic* | mmixware* | mon960* | lnews*) + ;; + linux-dietlibc) + os=linux-dietlibc + ;; + linux*) + os=`echo $os | sed -e 's|linux|linux-gnu|'` + ;; + lynx*178) + os=lynxos178 + ;; + lynx*5) + os=lynxos5 + ;; + lynx*) + os=lynxos + ;; + mac*) + os=`echo "$os" | sed -e 's|mac|macos|'` + ;; + opened*) + os=openedition + ;; + os400*) + os=os400 + ;; + sunos5*) + os=`echo "$os" | sed -e 's|sunos5|solaris2|'` + ;; + sunos6*) + os=`echo "$os" | sed -e 's|sunos6|solaris3|'` + ;; + wince*) + os=wince + ;; + utek*) + os=bsd + ;; + dynix*) + os=bsd + ;; + acis*) + os=aos + ;; + atheos*) + os=atheos + ;; + syllable*) + os=syllable + ;; + 386bsd) + os=bsd + ;; + ctix* | uts*) + os=sysv + ;; + nova*) + os=rtmk-nova + ;; + ns2) + os=nextstep2 + ;; + nsk*) + os=nsk + ;; + # Preserve the version number of sinix5. + sinix5.*) + os=`echo $os | sed -e 's|sinix|sysv|'` + ;; + sinix*) + os=sysv4 + ;; + tpf*) + os=tpf + ;; + triton*) + os=sysv3 + ;; + oss*) + os=sysv3 + ;; + svr4*) + os=sysv4 + ;; + svr3) + os=sysv3 + ;; + sysvr4) + os=sysv4 + ;; + # This must come after sysvr4. + sysv*) + ;; + ose*) + os=ose + ;; + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + os=mint + ;; + zvmoe) + os=zvmoe + ;; + dicos*) + os=dicos + ;; + pikeos*) + # Until real need of OS specific support for + # particular features comes up, bare metal + # configurations are quite functional. + case $basic_machine in + arm*) + os=eabi + ;; + *) + os=elf + ;; + esac + ;; + nacl*) + ;; + ios) + ;; + none) + ;; + *-eabi) + ;; + *) + echo Invalid configuration \`"$1"\': system \`"$os"\' not recognized 1>&2 + exit 1 + ;; +esac +else + +# Here we handle the default operating systems that come with various machines. +# The value should be what the vendor currently ships out the door with their +# machine or put another way, the most popular os provided with the machine. + +# Note that if you're going to try to match "-MANUFACTURER" here (say, +# "-sun"), then you have to tell the case statement up towards the top +# that MANUFACTURER isn't an operating system. Otherwise, code above +# will signal an error saying that MANUFACTURER isn't an operating +# system, and we'll never get to this point. + +case $basic_machine in + score-*) + os=elf + ;; + spu-*) + os=elf + ;; + *-acorn) + os=riscix1.2 + ;; + arm*-rebel) + os=linux + ;; + arm*-semi) + os=aout + ;; + c4x-* | tic4x-*) + os=coff + ;; + c8051-*) + os=elf + ;; + clipper-intergraph) + os=clix + ;; + hexagon-*) + os=elf + ;; + tic54x-*) + os=coff + ;; + tic55x-*) + os=coff + ;; + tic6x-*) + os=coff + ;; + # This must come before the *-dec entry. + pdp10-*) + os=tops20 + ;; + pdp11-*) + os=none + ;; + *-dec | vax-*) + os=ultrix4.2 + ;; + m68*-apollo) + os=domain + ;; + i386-sun) + os=sunos4.0.2 + ;; + m68000-sun) + os=sunos3 + ;; + m68*-cisco) + os=aout + ;; + mep-*) + os=elf + ;; + mips*-cisco) + os=elf + ;; + mips*-*) + os=elf + ;; + or32-*) + os=coff + ;; + *-tti) # must be before sparc entry or we get the wrong os. + os=sysv3 + ;; + sparc-* | *-sun) + os=sunos4.1.1 + ;; + pru-*) + os=elf + ;; + *-be) + os=beos + ;; + *-ibm) + os=aix + ;; + *-knuth) + os=mmixware + ;; + *-wec) + os=proelf + ;; + *-winbond) + os=proelf + ;; + *-oki) + os=proelf + ;; + *-hp) + os=hpux + ;; + *-hitachi) + os=hiux + ;; + i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) + os=sysv + ;; + *-cbm) + os=amigaos + ;; + *-dg) + os=dgux + ;; + *-dolphin) + os=sysv3 + ;; + m68k-ccur) + os=rtu + ;; + m88k-omron*) + os=luna + ;; + *-next) + os=nextstep + ;; + *-sequent) + os=ptx + ;; + *-crds) + os=unos + ;; + *-ns) + os=genix + ;; + i370-*) + os=mvs + ;; + *-gould) + os=sysv + ;; + *-highlevel) + os=bsd + ;; + *-encore) + os=bsd + ;; + *-sgi) + os=irix + ;; + *-siemens) + os=sysv4 + ;; + *-masscomp) + os=rtu + ;; + f30[01]-fujitsu | f700-fujitsu) + os=uxpv + ;; + *-rom68k) + os=coff + ;; + *-*bug) + os=coff + ;; + *-apple) + os=macos + ;; + *-atari*) + os=mint + ;; + *-wrs) + os=vxworks + ;; + *) + os=none + ;; +esac +fi + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +vendor=unknown +case $basic_machine in + *-unknown) + case $os in + riscix*) + vendor=acorn + ;; + sunos*) + vendor=sun + ;; + cnk*|-aix*) + vendor=ibm + ;; + beos*) + vendor=be + ;; + hpux*) + vendor=hp + ;; + mpeix*) + vendor=hp + ;; + hiux*) + vendor=hitachi + ;; + unos*) + vendor=crds + ;; + dgux*) + vendor=dg + ;; + luna*) + vendor=omron + ;; + genix*) + vendor=ns + ;; + clix*) + vendor=intergraph + ;; + mvs* | opened*) + vendor=ibm + ;; + os400*) + vendor=ibm + ;; + ptx*) + vendor=sequent + ;; + tpf*) + vendor=ibm + ;; + vxsim* | vxworks* | windiss*) + vendor=wrs + ;; + aux*) + vendor=apple + ;; + hms*) + vendor=hitachi + ;; + mpw* | macos*) + vendor=apple + ;; + *mint | mint[0-9]* | *MiNT | MiNT[0-9]*) + vendor=atari + ;; + vos*) + vendor=stratus + ;; + esac + basic_machine=`echo "$basic_machine" | sed "s/unknown/$vendor/"` + ;; +esac + +echo "$basic_machine-$os" +exit + +# Local variables: +# eval: (add-hook 'before-save-hook 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/third_party/nix/config/install-sh b/third_party/nix/config/install-sh new file mode 100755 index 000000000..377bb8687 --- /dev/null +++ b/third_party/nix/config/install-sh @@ -0,0 +1,527 @@ +#!/bin/sh +# install - install a program, script, or datafile + +scriptversion=2011-11-20.07; # UTC + +# This originates from X11R5 (mit/util/scripts/install.sh), which was +# later released in X11R6 (xc/config/util/install.sh) with the +# following copyright and license. +# +# Copyright (C) 1994 X Consortium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- +# TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# +# Except as contained in this notice, the name of the X Consortium shall not +# be used in advertising or otherwise to promote the sale, use or other deal- +# ings in this Software without prior written authorization from the X Consor- +# tium. +# +# +# FSF changes to this file are in the public domain. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# 'make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. + +nl=' +' +IFS=" "" $nl" + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit=${DOITPROG-} +if test -z "$doit"; then + doit_exec=exec +else + doit_exec=$doit +fi + +# Put in absolute file names if you don't have them in your path; +# or use environment vars. + +chgrpprog=${CHGRPPROG-chgrp} +chmodprog=${CHMODPROG-chmod} +chownprog=${CHOWNPROG-chown} +cmpprog=${CMPPROG-cmp} +cpprog=${CPPROG-cp} +mkdirprog=${MKDIRPROG-mkdir} +mvprog=${MVPROG-mv} +rmprog=${RMPROG-rm} +stripprog=${STRIPPROG-strip} + +posix_glob='?' +initialize_posix_glob=' + test "$posix_glob" != "?" || { + if (set -f) 2>/dev/null; then + posix_glob= + else + posix_glob=: + fi + } +' + +posix_mkdir= + +# Desired mode of installed file. +mode=0755 + +chgrpcmd= +chmodcmd=$chmodprog +chowncmd= +mvcmd=$mvprog +rmcmd="$rmprog -f" +stripcmd= + +src= +dst= +dir_arg= +dst_arg= + +copy_on_change=false +no_target_directory= + +usage="\ +Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE + or: $0 [OPTION]... SRCFILES... DIRECTORY + or: $0 [OPTION]... -t DIRECTORY SRCFILES... + or: $0 [OPTION]... -d DIRECTORIES... + +In the 1st form, copy SRCFILE to DSTFILE. +In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. +In the 4th, create DIRECTORIES. + +Options: + --help display this help and exit. + --version display version info and exit. + + -c (ignored) + -C install only if different (preserve the last data modification time) + -d create directories instead of installing files. + -g GROUP $chgrpprog installed files to GROUP. + -m MODE $chmodprog installed files to MODE. + -o USER $chownprog installed files to USER. + -s $stripprog installed files. + -t DIRECTORY install into DIRECTORY. + -T report an error if DSTFILE is a directory. + +Environment variables override the default commands: + CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG + RMPROG STRIPPROG +" + +while test $# -ne 0; do + case $1 in + -c) ;; + + -C) copy_on_change=true;; + + -d) dir_arg=true;; + + -g) chgrpcmd="$chgrpprog $2" + shift;; + + --help) echo "$usage"; exit $?;; + + -m) mode=$2 + case $mode in + *' '* | *' '* | *' +'* | *'*'* | *'?'* | *'['*) + echo "$0: invalid mode: $mode" >&2 + exit 1;; + esac + shift;; + + -o) chowncmd="$chownprog $2" + shift;; + + -s) stripcmd=$stripprog;; + + -t) dst_arg=$2 + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + shift;; + + -T) no_target_directory=true;; + + --version) echo "$0 $scriptversion"; exit $?;; + + --) shift + break;; + + -*) echo "$0: invalid option: $1" >&2 + exit 1;; + + *) break;; + esac + shift +done + +if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then + # When -d is used, all remaining arguments are directories to create. + # When -t is used, the destination is already specified. + # Otherwise, the last argument is the destination. Remove it from $@. + for arg + do + if test -n "$dst_arg"; then + # $@ is not empty: it contains at least $arg. + set fnord "$@" "$dst_arg" + shift # fnord + fi + shift # arg + dst_arg=$arg + # Protect names problematic for 'test' and other utilities. + case $dst_arg in + -* | [=\(\)!]) dst_arg=./$dst_arg;; + esac + done +fi + +if test $# -eq 0; then + if test -z "$dir_arg"; then + echo "$0: no input file specified." >&2 + exit 1 + fi + # It's OK to call 'install-sh -d' without argument. + # This can happen when creating conditional directories. + exit 0 +fi + +if test -z "$dir_arg"; then + do_exit='(exit $ret); exit $ret' + trap "ret=129; $do_exit" 1 + trap "ret=130; $do_exit" 2 + trap "ret=141; $do_exit" 13 + trap "ret=143; $do_exit" 15 + + # Set umask so as not to create temps with too-generous modes. + # However, 'strip' requires both read and write access to temps. + case $mode in + # Optimize common cases. + *644) cp_umask=133;; + *755) cp_umask=22;; + + *[0-7]) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw='% 200' + fi + cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; + *) + if test -z "$stripcmd"; then + u_plus_rw= + else + u_plus_rw=,u+rw + fi + cp_umask=$mode$u_plus_rw;; + esac +fi + +for src +do + # Protect names problematic for 'test' and other utilities. + case $src in + -* | [=\(\)!]) src=./$src;; + esac + + if test -n "$dir_arg"; then + dst=$src + dstdir=$dst + test -d "$dstdir" + dstdir_status=$? + else + + # Waiting for this to be detected by the "$cpprog $src $dsttmp" command + # might cause directories to be created, which would be especially bad + # if $src (and thus $dsttmp) contains '*'. + if test ! -f "$src" && test ! -d "$src"; then + echo "$0: $src does not exist." >&2 + exit 1 + fi + + if test -z "$dst_arg"; then + echo "$0: no destination specified." >&2 + exit 1 + fi + dst=$dst_arg + + # If destination is a directory, append the input filename; won't work + # if double slashes aren't ignored. + if test -d "$dst"; then + if test -n "$no_target_directory"; then + echo "$0: $dst_arg: Is a directory" >&2 + exit 1 + fi + dstdir=$dst + dst=$dstdir/`basename "$src"` + dstdir_status=0 + else + # Prefer dirname, but fall back on a substitute if dirname fails. + dstdir=` + (dirname "$dst") 2>/dev/null || + expr X"$dst" : 'X\(.*[^/]\)//*[^/][^/]*/*$' \| \ + X"$dst" : 'X\(//\)[^/]' \| \ + X"$dst" : 'X\(//\)$' \| \ + X"$dst" : 'X\(/\)' \| . 2>/dev/null || + echo X"$dst" | + sed '/^X\(.*[^/]\)\/\/*[^/][^/]*\/*$/{ + s//\1/ + q + } + /^X\(\/\/\)[^/].*/{ + s//\1/ + q + } + /^X\(\/\/\)$/{ + s//\1/ + q + } + /^X\(\/\).*/{ + s//\1/ + q + } + s/.*/./; q' + ` + + test -d "$dstdir" + dstdir_status=$? + fi + fi + + obsolete_mkdir_used=false + + if test $dstdir_status != 0; then + case $posix_mkdir in + '') + # Create intermediate dirs using mode 755 as modified by the umask. + # This is like FreeBSD 'install' as of 1997-10-28. + umask=`umask` + case $stripcmd.$umask in + # Optimize common cases. + *[2367][2367]) mkdir_umask=$umask;; + .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; + + *[0-7]) + mkdir_umask=`expr $umask + 22 \ + - $umask % 100 % 40 + $umask % 20 \ + - $umask % 10 % 4 + $umask % 2 + `;; + *) mkdir_umask=$umask,go-w;; + esac + + # With -d, create the new directory with the user-specified mode. + # Otherwise, rely on $mkdir_umask. + if test -n "$dir_arg"; then + mkdir_mode=-m$mode + else + mkdir_mode= + fi + + posix_mkdir=false + case $umask in + *[123567][0-7][0-7]) + # POSIX mkdir -p sets u+wx bits regardless of umask, which + # is incompatible with FreeBSD 'install' when (umask & 300) != 0. + ;; + *) + tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ + trap 'ret=$?; rmdir "$tmpdir/d" "$tmpdir" 2>/dev/null; exit $ret' 0 + + if (umask $mkdir_umask && + exec $mkdirprog $mkdir_mode -p -- "$tmpdir/d") >/dev/null 2>&1 + then + if test -z "$dir_arg" || { + # Check for POSIX incompatibilities with -m. + # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or + # other-writable bit of parent directory when it shouldn't. + # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. + ls_ld_tmpdir=`ls -ld "$tmpdir"` + case $ls_ld_tmpdir in + d????-?r-*) different_mode=700;; + d????-?--*) different_mode=755;; + *) false;; + esac && + $mkdirprog -m$different_mode -p -- "$tmpdir" && { + ls_ld_tmpdir_1=`ls -ld "$tmpdir"` + test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" + } + } + then posix_mkdir=: + fi + rmdir "$tmpdir/d" "$tmpdir" + else + # Remove any dirs left behind by ancient mkdir implementations. + rmdir ./$mkdir_mode ./-p ./-- 2>/dev/null + fi + trap '' 0;; + esac;; + esac + + if + $posix_mkdir && ( + umask $mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" + ) + then : + else + + # The umask is ridiculous, or mkdir does not conform to POSIX, + # or it failed possibly due to a race condition. Create the + # directory the slow way, step by step, checking for races as we go. + + case $dstdir in + /*) prefix='/';; + [-=\(\)!]*) prefix='./';; + *) prefix='';; + esac + + eval "$initialize_posix_glob" + + oIFS=$IFS + IFS=/ + $posix_glob set -f + set fnord $dstdir + shift + $posix_glob set +f + IFS=$oIFS + + prefixes= + + for d + do + test X"$d" = X && continue + + prefix=$prefix$d + if test -d "$prefix"; then + prefixes= + else + if $posix_mkdir; then + (umask=$mkdir_umask && + $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break + # Don't fail if two instances are running concurrently. + test -d "$prefix" || exit 1 + else + case $prefix in + *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; + *) qprefix=$prefix;; + esac + prefixes="$prefixes '$qprefix'" + fi + fi + prefix=$prefix/ + done + + if test -n "$prefixes"; then + # Don't fail if two instances are running concurrently. + (umask $mkdir_umask && + eval "\$doit_exec \$mkdirprog $prefixes") || + test -d "$dstdir" || exit 1 + obsolete_mkdir_used=true + fi + fi + fi + + if test -n "$dir_arg"; then + { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && + { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || + test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 + else + + # Make a couple of temp file names in the proper directory. + dsttmp=$dstdir/_inst.$$_ + rmtmp=$dstdir/_rm.$$_ + + # Trap to clean up those temp files at exit. + trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 + + # Copy the file name to the temp name. + (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && + + # and set any options; do chmod last to preserve setuid bits. + # + # If any of these fail, we abort the whole thing. If we want to + # ignore errors from any of these, just make sure not to ignore + # errors from the above "$doit $cpprog $src $dsttmp" command. + # + { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && + { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && + { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && + { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && + + # If -C, don't bother to copy if it wouldn't change the file. + if $copy_on_change && + old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && + new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && + + eval "$initialize_posix_glob" && + $posix_glob set -f && + set X $old && old=:$2:$4:$5:$6 && + set X $new && new=:$2:$4:$5:$6 && + $posix_glob set +f && + + test "$old" = "$new" && + $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 + then + rm -f "$dsttmp" + else + # Rename the file to the real destination. + $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || + + # The rename failed, perhaps because mv can't rename something else + # to itself, or perhaps because mv is so ancient that it does not + # support -f. + { + # Now remove or move aside any old file at destination location. + # We try this two ways since rm can't unlink itself on some + # systems and the destination file might be busy for other + # reasons. In this case, the final cleanup might fail but the new + # file should still install successfully. + { + test ! -f "$dst" || + $doit $rmcmd -f "$dst" 2>/dev/null || + { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && + { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } + } || + { echo "$0: cannot unlink or rename $dst" >&2 + (exit 1); exit 1 + } + } && + + # Now rename the file to the real destination. + $doit $mvcmd "$dsttmp" "$dst" + } + fi || exit 1 + + trap '' 0 + fi +done + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "scriptversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/third_party/nix/configure.ac b/third_party/nix/configure.ac new file mode 100644 index 000000000..a52830b38 --- /dev/null +++ b/third_party/nix/configure.ac @@ -0,0 +1,316 @@ +AC_INIT(nix, m4_esyscmd([bash -c "echo -n $(cat ./.version)$VERSION_SUFFIX"])) +AC_CONFIG_MACRO_DIRS([m4]) +AC_CONFIG_SRCDIR(README.md) +AC_CONFIG_AUX_DIR(config) + +AC_PROG_SED + +# Construct a Nix system name (like "i686-linux"). +AC_CANONICAL_HOST +AC_MSG_CHECKING([for the canonical Nix system name]) + +AC_ARG_WITH(system, AC_HELP_STRING([--with-system=SYSTEM], + [Platform identifier (e.g., `i686-linux').]), + [system=$withval], + [case "$host_cpu" in + i*86) + machine_name="i686";; + amd64) + machine_name="x86_64";; + armv6|armv7) + machine_name="${host_cpu}l";; + *) + machine_name="$host_cpu";; + esac + + case "$host_os" in + linux-gnu*|linux-musl*) + # For backward compatibility, strip the `-gnu' part. + system="$machine_name-linux";; + *) + # Strip the version number from names such as `gnu0.3', + # `darwin10.2.0', etc. + system="$machine_name-`echo $host_os | "$SED" -e's/@<:@0-9.@:>@*$//g'`";; + esac]) + +sys_name=$(uname -s | tr 'A-Z ' 'a-z_') + +case $sys_name in + cygwin*) + sys_name=cygwin + ;; +esac + +AC_MSG_RESULT($system) +AC_SUBST(system) +AC_DEFINE_UNQUOTED(SYSTEM, ["$system"], [platform identifier ('cpu-os')]) + + +# State should be stored in /nix/var, unless the user overrides it explicitly. +test "$localstatedir" = '${prefix}/var' && localstatedir=/nix/var + + +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} +AC_PROG_CC +AC_PROG_CXX +AC_PROG_CPP +AX_CXX_COMPILE_STDCXX_17([noext], [mandatory]) + +AC_CHECK_TOOL([AR], [ar]) + +# Use 64-bit file system calls so that we can support files > 2 GiB. +AC_SYS_LARGEFILE + + +# Solaris-specific stuff. +AC_STRUCT_DIRENT_D_TYPE +if test "$sys_name" = sunos; then + # Solaris requires -lsocket -lnsl for network functions + LIBS="-lsocket -lnsl $LIBS" +fi + + +# Check for pubsetbuf. +AC_MSG_CHECKING([for pubsetbuf]) +AC_LANG_PUSH(C++) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include +using namespace std; +static char buf[1024];]], + [[cerr.rdbuf()->pubsetbuf(buf, sizeof(buf));]])], + [AC_MSG_RESULT(yes) AC_DEFINE(HAVE_PUBSETBUF, 1, [Whether pubsetbuf is available.])], + AC_MSG_RESULT(no)) +AC_LANG_POP(C++) + + +AC_CHECK_FUNCS([statvfs pipe2]) + + +# Check for lutimes, optionally used for changing the mtime of +# symlinks. +AC_CHECK_FUNCS([lutimes]) + + +# Check whether the store optimiser can optimise symlinks. +AC_MSG_CHECKING([whether it is possible to create a link to a symlink]) +ln -s bla tmp_link +if ln tmp_link tmp_link2 2> /dev/null; then + AC_MSG_RESULT(yes) + AC_DEFINE(CAN_LINK_SYMLINK, 1, [Whether link() works on symlinks.]) +else + AC_MSG_RESULT(no) +fi +rm -f tmp_link tmp_link2 + + +# Check for . +AC_LANG_PUSH(C++) +AC_CHECK_HEADERS([locale]) +AC_LANG_POP(C++) + + +AC_DEFUN([NEED_PROG], +[ +AC_PATH_PROG($1, $2) +if test -z "$$1"; then + AC_MSG_ERROR([$2 is required]) +fi +]) + +NEED_PROG(bash, bash) +NEED_PROG(patch, patch) +AC_PATH_PROG(xmllint, xmllint, false) +AC_PATH_PROG(xsltproc, xsltproc, false) +AC_PATH_PROG(flex, flex, false) +AC_PATH_PROG(bison, bison, false) +NEED_PROG(sed, sed) +NEED_PROG(tar, tar) +NEED_PROG(bzip2, bzip2) +NEED_PROG(gzip, gzip) +NEED_PROG(xz, xz) +AC_PATH_PROG(dot, dot) +AC_PATH_PROG(lsof, lsof, lsof) + + +NEED_PROG(cat, cat) +NEED_PROG(tr, tr) +AC_ARG_WITH(coreutils-bin, AC_HELP_STRING([--with-coreutils-bin=PATH], + [path of cat, mkdir, etc.]), + coreutils=$withval, coreutils=$(dirname $cat)) +AC_SUBST(coreutils) + + +AC_ARG_WITH(store-dir, AC_HELP_STRING([--with-store-dir=PATH], + [path of the Nix store (defaults to /nix/store)]), + storedir=$withval, storedir='/nix/store') +AC_SUBST(storedir) + + +# Look for boost, a required dependency. +# Note that AX_BOOST_BASE only exports *CPP* BOOST_CPPFLAGS, no CXX flags, +# and CPPFLAGS are not passed to the C++ compiler automatically. +# Thus we append the returned CPPFLAGS to the CXXFLAGS here. +AX_BOOST_BASE([1.66], [CXXFLAGS="$BOOST_CPPFLAGS $CXXFLAGS"], [AC_MSG_ERROR([Nix requires boost.])]) +# For unknown reasons, setting this directly in the ACTION-IF-FOUND above +# ends up with LDFLAGS being empty, so we set it afterwards. +LDFLAGS="$BOOST_LDFLAGS $LDFLAGS" + + +# Look for OpenSSL, a required dependency. +PKG_CHECK_MODULES([OPENSSL], [libcrypto], [CXXFLAGS="$OPENSSL_CFLAGS $CXXFLAGS"]) + + +# Look for libbz2, a required dependency. +AC_CHECK_LIB([bz2], [BZ2_bzWriteOpen], [true], + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) +AC_CHECK_HEADERS([bzlib.h], [true], + [AC_MSG_ERROR([Nix requires libbz2, which is part of bzip2. See https://web.archive.org/web/20180624184756/http://www.bzip.org/.])]) + + +# Look for SQLite, a required dependency. +PKG_CHECK_MODULES([SQLITE3], [sqlite3 >= 3.6.19], [CXXFLAGS="$SQLITE3_CFLAGS $CXXFLAGS"]) + + +# Look for libcurl, a required dependency. +PKG_CHECK_MODULES([LIBCURL], [libcurl], [CXXFLAGS="$LIBCURL_CFLAGS $CXXFLAGS"]) + +# Look for editline, a required dependency. +# The the libeditline.pc file was added only in libeditline >= 1.15.2, +# see https://github.com/troglobit/editline/commit/0a8f2ef4203c3a4a4726b9dd1336869cd0da8607, +# but e.g. Ubuntu 16.04 has an older version, so we fall back to searching for +# editline.h when the pkg-config approach fails. +PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLAGS"], [ + AC_CHECK_HEADERS([editline.h], [true], + [AC_MSG_ERROR([Nix requires libeditline; it was found neither via pkg-config nor its normal header.])]) + AC_SEARCH_LIBS([readline read_history], [editline], [], + [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) +]) + +# Look for libsodium, an optional dependency. +PKG_CHECK_MODULES([SODIUM], [libsodium], + [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) + CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" + have_sodium=1], [have_sodium=]) +AC_SUBST(HAVE_SODIUM, [$have_sodium]) + + +# Look for liblzma, a required dependency. +PKG_CHECK_MODULES([LIBLZMA], [liblzma], [CXXFLAGS="$LIBLZMA_CFLAGS $CXXFLAGS"]) +AC_CHECK_LIB([lzma], [lzma_stream_encoder_mt], + [AC_DEFINE([HAVE_LZMA_MT], [1], [xz multithreaded compression support])]) + + +# Look for libbrotli{enc,dec}. +PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) + + +# Look for libseccomp, required for Linux sandboxing. +if test "$sys_name" = linux; then + AC_ARG_ENABLE([seccomp-sandboxing], + AC_HELP_STRING([--disable-seccomp-sandboxing], + [Don't build support for seccomp sandboxing (only recommended if your arch doesn't support libseccomp yet!)] + )) + if test "x$enable_seccomp_sandboxing" != "xno"; then + PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + have_seccomp=1 + AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) + else + have_seccomp= + fi +else + have_seccomp= +fi +AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) + + +# Look for aws-cpp-sdk-s3. +AC_LANG_PUSH(C++) +AC_CHECK_HEADERS([aws/s3/S3Client.h], + [AC_DEFINE([ENABLE_S3], [1], [Whether to enable S3 support via aws-sdk-cpp.]) + enable_s3=1], [enable_s3=]) +AC_SUBST(ENABLE_S3, [$enable_s3]) +AC_LANG_POP(C++) + +if test -n "$enable_s3"; then + declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) + AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) + AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) +fi + + +# Whether to use the Boehm garbage collector. +AC_ARG_ENABLE(gc, AC_HELP_STRING([--enable-gc], + [enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=no]]), + gc=$enableval, gc=no) +if test "$gc" = yes; then + PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) + CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" + AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) +fi + + +# documentation generation switch +AC_ARG_ENABLE(doc-gen, AC_HELP_STRING([--disable-doc-gen], + [disable documentation generation]), + doc_generate=$enableval, doc_generate=yes) +AC_SUBST(doc_generate) + + +# Setuid installations. +AC_CHECK_FUNCS([setresuid setreuid lchown]) + + +# Nice to have, but not essential. +AC_CHECK_FUNCS([strsignal posix_fallocate sysconf]) + + +# This is needed if bzip2 is a static library, and the Nix libraries +# are dynamic. +if test "$(uname)" = "Darwin"; then + LDFLAGS="-all_load $LDFLAGS" +fi + + +# Do we have GNU tar? +AC_MSG_CHECKING([if you have a recent GNU tar]) +if $tar --version 2> /dev/null | grep -q GNU && tar cvf /dev/null --warning=no-timestamp ./config.log > /dev/null; then + AC_MSG_RESULT(yes) + tarFlags="--warning=no-timestamp" +else + AC_MSG_RESULT(no) +fi +AC_SUBST(tarFlags) + + +AC_ARG_WITH(sandbox-shell, AC_HELP_STRING([--with-sandbox-shell=PATH], + [path of a statically-linked shell to use as /bin/sh in sandboxes]), + sandbox_shell=$withval) +AC_SUBST(sandbox_shell) + +AC_ARG_ENABLE(shared, AC_HELP_STRING([--enable-shared], + [Build shared libraries for Nix [default=yes]]), + shared=$enableval, shared=yes) +if test "$shared" = yes; then + AC_SUBST(BUILD_SHARED_LIBS, 1, [Whether to build shared libraries.]) +else + AC_SUBST(BUILD_SHARED_LIBS, 0, [Whether to build shared libraries.]) +fi + + +# Expand all variables in config.status. +test "$prefix" = NONE && prefix=$ac_default_prefix +test "$exec_prefix" = NONE && exec_prefix='${prefix}' +for name in $ac_subst_vars; do + declare $name="$(eval echo "${!name}")" + declare $name="$(eval echo "${!name}")" + declare $name="$(eval echo "${!name}")" +done + +rm -f Makefile.config + +AC_CONFIG_HEADER([config.h]) +AC_CONFIG_FILES([]) +AC_OUTPUT diff --git a/third_party/nix/contrib/stack-collapse.py b/third_party/nix/contrib/stack-collapse.py new file mode 100755 index 000000000..f5602c95c --- /dev/null +++ b/third_party/nix/contrib/stack-collapse.py @@ -0,0 +1,38 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p python3 --pure + +# To be used with `--trace-function-calls` and `flamegraph.pl`. +# +# For example: +# +# nix-instantiate --trace-function-calls '' -A hello 2> nix-function-calls.trace +# ./contrib/stack-collapse.py nix-function-calls.trace > nix-function-calls.folded +# nix-shell -p flamegraph --run "flamegraph.pl nix-function-calls.folded > nix-function-calls.svg" + +import sys +from pprint import pprint +import fileinput + +stack = [] +timestack = [] + +for line in fileinput.input(): + components = line.strip().split(" ", 2) + if components[0] != "function-trace": + continue + + direction = components[1] + components = components[2].rsplit(" ", 2) + + loc = components[0] + _at = components[1] + time = int(components[2]) + + if direction == "entered": + stack.append(loc) + timestack.append(time) + elif direction == "exited": + dur = time - timestack.pop() + vst = ";".join(stack) + print(f"{vst} {dur}") + stack.pop() diff --git a/third_party/nix/corepkgs/buildenv.nix b/third_party/nix/corepkgs/buildenv.nix new file mode 100644 index 000000000..0bac4c44b --- /dev/null +++ b/third_party/nix/corepkgs/buildenv.nix @@ -0,0 +1,25 @@ +{ derivations, manifest }: + +derivation { + name = "user-environment"; + system = "builtin"; + builder = "builtin:buildenv"; + + inherit manifest; + + # !!! grmbl, need structured data for passing this in a clean way. + derivations = + map (d: + [ (d.meta.active or "true") + (d.meta.priority or 5) + (builtins.length d.outputs) + ] ++ map (output: builtins.getAttr output d) d.outputs) + derivations; + + # Building user environments remotely just causes huge amounts of + # network traffic, so don't do that. + preferLocalBuild = true; + + # Also don't bother substituting. + allowSubstitutes = false; +} diff --git a/third_party/nix/corepkgs/config.nix.in b/third_party/nix/corepkgs/config.nix.in new file mode 100644 index 000000000..32ce6b399 --- /dev/null +++ b/third_party/nix/corepkgs/config.nix.in @@ -0,0 +1,29 @@ +let + fromEnv = var: def: + let val = builtins.getEnv var; in + if val != "" then val else def; +in rec { + shell = "@bash@"; + coreutils = "@coreutils@"; + bzip2 = "@bzip2@"; + gzip = "@gzip@"; + xz = "@xz@"; + tar = "@tar@"; + tarFlags = "@tarFlags@"; + tr = "@tr@"; + nixBinDir = fromEnv "NIX_BIN_DIR" "@bindir@"; + nixPrefix = "@prefix@"; + nixLibexecDir = fromEnv "NIX_LIBEXEC_DIR" "@libexecdir@"; + nixLocalstateDir = "@localstatedir@"; + nixSysconfDir = "@sysconfdir@"; + nixStoreDir = fromEnv "NIX_STORE_DIR" "@storedir@"; + + # If Nix is installed in the Nix store, then automatically add it as + # a dependency to the core packages. This ensures that they work + # properly in a chroot. + chrootDeps = + if dirOf nixPrefix == builtins.storeDir then + [ (builtins.storePath nixPrefix) ] + else + [ ]; +} diff --git a/third_party/nix/corepkgs/derivation.nix b/third_party/nix/corepkgs/derivation.nix new file mode 100644 index 000000000..c0fbe8082 --- /dev/null +++ b/third_party/nix/corepkgs/derivation.nix @@ -0,0 +1,27 @@ +/* This is the implementation of the ‘derivation’ builtin function. + It's actually a wrapper around the ‘derivationStrict’ primop. */ + +drvAttrs @ { outputs ? [ "out" ], ... }: + +let + + strict = derivationStrict drvAttrs; + + commonAttrs = drvAttrs // (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvAttrs; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName strict; + drvPath = strict.drvPath; + type = "derivation"; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/third_party/nix/corepkgs/fetchurl.nix b/third_party/nix/corepkgs/fetchurl.nix new file mode 100644 index 000000000..a84777f57 --- /dev/null +++ b/third_party/nix/corepkgs/fetchurl.nix @@ -0,0 +1,41 @@ +{ system ? "" # obsolete +, url +, hash ? "" # an SRI ash + +# Legacy hash specification +, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" +, outputHash ? + if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 +, outputHashAlgo ? + if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + +, executable ? false +, unpack ? false +, name ? baseNameOf (toString url) +}: + +derivation { + builder = "builtin:fetchurl"; + + # New-style output content requirements. + inherit outputHashAlgo outputHash; + outputHashMode = if unpack || executable then "recursive" else "flat"; + + inherit name url executable unpack; + + system = "builtin"; + + # No need to double the amount of network traffic + preferLocalBuild = true; + + impureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + # To make "nix-prefetch-url" work. + urls = [ url ]; +} diff --git a/third_party/nix/corepkgs/imported-drv-to-derivation.nix b/third_party/nix/corepkgs/imported-drv-to-derivation.nix new file mode 100644 index 000000000..eab8b050e --- /dev/null +++ b/third_party/nix/corepkgs/imported-drv-to-derivation.nix @@ -0,0 +1,21 @@ +attrs @ { drvPath, outputs, name, ... }: + +let + + commonAttrs = (builtins.listToAttrs outputsList) // + { all = map (x: x.value) outputsList; + inherit drvPath name; + type = "derivation"; + }; + + outputToAttrListElement = outputName: + { name = outputName; + value = commonAttrs // { + outPath = builtins.getAttr outputName attrs; + inherit outputName; + }; + }; + + outputsList = map outputToAttrListElement outputs; + +in (builtins.head outputsList).value diff --git a/third_party/nix/corepkgs/local.mk b/third_party/nix/corepkgs/local.mk new file mode 100644 index 000000000..362c8eb61 --- /dev/null +++ b/third_party/nix/corepkgs/local.mk @@ -0,0 +1,5 @@ +corepkgs_FILES = buildenv.nix unpack-channel.nix derivation.nix fetchurl.nix imported-drv-to-derivation.nix + +$(foreach file,config.nix $(corepkgs_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/corepkgs))) + +template-files += $(d)/config.nix diff --git a/third_party/nix/corepkgs/unpack-channel.nix b/third_party/nix/corepkgs/unpack-channel.nix new file mode 100644 index 000000000..d39a20637 --- /dev/null +++ b/third_party/nix/corepkgs/unpack-channel.nix @@ -0,0 +1,39 @@ +with import ; + +let + + builder = builtins.toFile "unpack-channel.sh" + '' + mkdir $out + cd $out + xzpat="\.xz\$" + gzpat="\.gz\$" + if [[ "$src" =~ $xzpat ]]; then + ${xz} -d < $src | ${tar} xf - ${tarFlags} + elif [[ "$src" =~ $gzpat ]]; then + ${gzip} -d < $src | ${tar} xf - ${tarFlags} + else + ${bzip2} -d < $src | ${tar} xf - ${tarFlags} + fi + if [ * != $channelName ]; then + mv * $out/$channelName + fi + ''; + +in + +{ name, channelName, src }: + +derivation { + system = builtins.currentSystem; + builder = shell; + args = [ "-e" builder ]; + inherit name channelName src; + + PATH = "${nixBinDir}:${coreutils}"; + + # No point in doing this remotely. + preferLocalBuild = true; + + inherit chrootDeps; +} diff --git a/third_party/nix/doc/manual/advanced-topics/advanced-topics.xml b/third_party/nix/doc/manual/advanced-topics/advanced-topics.xml new file mode 100644 index 000000000..871b7eb1d --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/advanced-topics.xml @@ -0,0 +1,14 @@ + + +Advanced Topics + + + + + + + diff --git a/third_party/nix/doc/manual/advanced-topics/cores-vs-jobs.xml b/third_party/nix/doc/manual/advanced-topics/cores-vs-jobs.xml new file mode 100644 index 000000000..eba645faf --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/cores-vs-jobs.xml @@ -0,0 +1,121 @@ + + +Tuning Cores and Jobs + +Nix has two relevant settings with regards to how your CPU cores +will be utilized: and +. This chapter will talk about what +they are, how they interact, and their configuration trade-offs. + + + + + + Dictates how many separate derivations will be built at the same + time. If you set this to zero, the local machine will do no + builds. Nix will still substitute from binary caches, and build + remotely if remote builders are configured. + + + + + + Suggests how many cores each derivation should use. Similar to + make -j. + + + + +The setting determines the value of +NIX_BUILD_CORES. NIX_BUILD_CORES is equal +to , unless +equals 0, in which case NIX_BUILD_CORES +will be the total number of cores in the system. + +The total number of consumed cores is a simple multiplication, + * NIX_BUILD_CORES. + +The balance on how to set these two independent variables depends +upon each builder's workload and hardware. Here are a few example +scenarios on a machine with 24 cores: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Balancing 24 Build Cores
NIX_BUILD_CORESMaximum ProcessesResult
1242424 + One derivation will be built at a time, each one can use 24 + cores. Undersold if a job can’t use 24 cores. +
46624 + Four derivations will be built at once, each given access to + six cores. +
126672 + 12 derivations will be built at once, each given access to six + cores. This configuration is over-sold. If all 12 derivations + being built simultaneously try to use all six cores, the + machine's performance will be degraded due to extensive context + switching between the 12 builds. +
241124 + 24 derivations can build at the same time, each using a single + core. Never oversold, but derivations which require many cores + will be very slow to compile. +
24024576 + 24 derivations can build at the same time, each using all the + available cores of the machine. Very likely to be oversold, + and very likely to suffer context switches. +
+ +It is up to the derivations' build script to respect +host's requested cores-per-build by following the value of the +NIX_BUILD_CORES environment variable. + + diff --git a/third_party/nix/doc/manual/advanced-topics/diff-hook.xml b/third_party/nix/doc/manual/advanced-topics/diff-hook.xml new file mode 100644 index 000000000..fb4bf819f --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/diff-hook.xml @@ -0,0 +1,205 @@ + + +Verifying Build Reproducibility with <option linkend="conf-diff-hook">diff-hook</option> + +Check build reproducibility by running builds multiple times +and comparing their results. + +Specify a program with Nix's to +compare build results when two builds produce different results. Note: +this hook is only executed if the results are not the same, this hook +is not used for determining if the results are the same. + +For purposes of demonstration, we'll use the following Nix file, +deterministic.nix for testing: + + +let + inherit (import <nixpkgs> {}) runCommand; +in { + stable = runCommand "stable" {} '' + touch $out + ''; + + unstable = runCommand "unstable" {} '' + echo $RANDOM > $out + ''; +} + + +Additionally, nix.conf contains: + + +diff-hook = /etc/nix/my-diff-hook +run-diff-hook = true + + +where /etc/nix/my-diff-hook is an executable +file containing: + + +#!/bin/sh +exec >&2 +echo "For derivation $3:" +/run/current-system/sw/bin/diff -r "$1" "$2" + + + + +The diff hook is executed by the same user and group who ran the +build. However, the diff hook does not have write access to the store +path just built. + +
+ + Spot-Checking Build Determinism + + + + Verify a path which already exists in the Nix store by passing + to the build command. + + + If the build passes and is deterministic, Nix will exit with a + status code of 0: + + +$ nix-build ./deterministic.nix -A stable +these derivations will be built: + /nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv +building '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + +$ nix-build ./deterministic.nix -A stable --check +checking outputs of '/nix/store/z98fasz2jqy9gs0xbvdj939p27jwda38-stable.drv'... +/nix/store/yyxlzw3vqaas7wfp04g0b1xg51f2czgq-stable + + + If the build is not deterministic, Nix will exit with a status + code of 1: + + +$ nix-build ./deterministic.nix -A unstable +these derivations will be built: + /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv +building '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable + +$ nix-build ./deterministic.nix -A unstable --check +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs + + +In the Nix daemon's log, we will now see: + +For derivation /nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv: +1c1 +< 8108 +--- +> 30204 + + + + Using with + will cause Nix to keep the second build's output in a special, + .check path: + + +$ nix-build ./deterministic.nix -A unstable --check --keep-failed +checking outputs of '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv'... +note: keeping build directory '/tmp/nix-build-unstable.drv-0' +error: derivation '/nix/store/cgl13lbj1w368r5z8gywipl1ifli7dhk-unstable.drv' may not be deterministic: output '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable' differs from '/nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check' + + + In particular, notice the + /nix/store/krpqk0l9ib0ibi1d2w52z293zw455cap-unstable.check + output. Nix has copied the build results to that directory where you + can examine it. + + + <literal>.check</literal> paths are not registered store paths + + Check paths are not protected against garbage collection, + and this path will be deleted on the next garbage collection. + + The path is guaranteed to be alive for the duration of + 's execution, but may be deleted + any time after. + + If the comparison is performed as part of automated tooling, + please use the diff-hook or author your tooling to handle the case + where the build was not deterministic and also a check path does + not exist. + + + + is only usable if the derivation has + been built on the system already. If the derivation has not been + built Nix will fail with the error: + +error: some outputs of '/nix/store/hzi1h60z2qf0nb85iwnpvrai3j2w7rr6-unstable.drv' are not valid, so checking is not possible + + + Run the build without , and then try with + again. + +
+ +
+ + Automatic and Optionally Enforced Determinism Verification + + + + Automatically verify every build at build time by executing the + build multiple times. + + + + Setting and + in your + nix.conf permits the automated verification + of every build Nix performs. + + + + The following configuration will run each build three times, and + will require the build to be deterministic: + + +enforce-determinism = true +repeat = 2 + + + + + Setting to false as in + the following configuration will run the build multiple times, + execute the build hook, but will allow the build to succeed even + if it does not build reproducibly: + + +enforce-determinism = false +repeat = 1 + + + + + An example output of this configuration: + +$ nix-build ./test.nix -A unstable +these derivations will be built: + /nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 1/2)... +building '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' (round 2/2)... +output '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable' of '/nix/store/ch6llwpr2h8c3jmnf3f2ghkhx59aa97f-unstable.drv' differs from '/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable.check' from previous round +/nix/store/6xg356v9gl03hpbbg8gws77n19qanh02-unstable + + +
+ diff --git a/third_party/nix/doc/manual/advanced-topics/distributed-builds.xml b/third_party/nix/doc/manual/advanced-topics/distributed-builds.xml new file mode 100644 index 000000000..9ac4a92cd --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/distributed-builds.xml @@ -0,0 +1,190 @@ + + +Remote Builds + +Nix supports remote builds, where a local Nix installation can +forward Nix builds to other machines. This allows multiple builds to +be performed in parallel and allows Nix to perform multi-platform +builds in a semi-transparent way. For instance, if you perform a +build for a x86_64-darwin on an +i686-linux machine, Nix can automatically forward +the build to a x86_64-darwin machine, if +available. + +To forward a build to a remote machine, it’s required that the +remote machine is accessible via SSH and that it has Nix +installed. You can test whether connecting to the remote Nix instance +works, e.g. + + +$ nix ping-store --store ssh://mac + + +will try to connect to the machine named mac. It is +possible to specify an SSH identity file as part of the remote store +URI, e.g. + + +$ nix ping-store --store ssh://mac?ssh-key=/home/alice/my-key + + +Since builds should be non-interactive, the key should not have a +passphrase. Alternatively, you can load identities ahead of time into +ssh-agent or gpg-agent. + +If you get the error + + +bash: nix-store: command not found +error: cannot connect to 'mac' + + +then you need to ensure that the PATH of +non-interactive login shells contains Nix. + +If you are building via the Nix daemon, it is the Nix +daemon user account (that is, root) that should +have SSH access to the remote machine. If you can’t or don’t want to +configure root to be able to access to remote +machine, you can use a private Nix store instead by passing +e.g. --store ~/my-nix. + +The list of remote machines can be specified on the command line +or in the Nix configuration file. The former is convenient for +testing. For example, the following command allows you to build a +derivation for x86_64-darwin on a Linux machine: + + +$ uname +Linux + +$ nix build \ + '(with import <nixpkgs> { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ + --builders 'ssh://mac x86_64-darwin' +[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac + +$ cat ./result +Darwin + + +It is possible to specify multiple builders separated by a semicolon +or a newline, e.g. + + + --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' + + + +Each machine specification consists of the following elements, +separated by spaces. Only the first element is required. +To leave a field at its default, set it to -. + + + + The URI of the remote store in the format + ssh://[username@]hostname, + e.g. ssh://nix@mac or + ssh://mac. For backward compatibility, + ssh:// may be omitted. The hostname may be an + alias defined in your + ~/.ssh/config. + + A comma-separated list of Nix platform type + identifiers, such as x86_64-darwin. It is + possible for a machine to support multiple platform types, e.g., + i686-linux,x86_64-linux. If omitted, this + defaults to the local platform type. + + The SSH identity file to be used to log in to the + remote machine. If omitted, SSH will use its regular + identities. + + The maximum number of builds that Nix will execute + in parallel on the machine. Typically this should be equal to the + number of CPU cores. For instance, the machine + itchy in the example will execute up to 8 builds + in parallel. + + The “speed factor”, indicating the relative speed of + the machine. If there are multiple machines of the right type, Nix + will prefer the fastest, taking load into account. + + A comma-separated list of supported + features. If a derivation has the + requiredSystemFeatures attribute, then Nix will + only perform the derivation on a machine that has the specified + features. For instance, the attribute + + +requiredSystemFeatures = [ "kvm" ]; + + + will cause the build to be performed on a machine that has the + kvm feature. + + A comma-separated list of mandatory + features. A machine will only be used to build a + derivation if all of the machine’s mandatory features appear in the + derivation’s requiredSystemFeatures + attribute.. + + + +For example, the machine specification + + +nix@scratchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 1 kvm +nix@itchy.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 8 2 +nix@poochie.labs.cs.uu.nl i686-linux /home/nix/.ssh/id_scratchy_auto 1 2 kvm benchmark + + +specifies several machines that can perform +i686-linux builds. However, +poochie will only do builds that have the attribute + + +requiredSystemFeatures = [ "benchmark" ]; + + +or + + +requiredSystemFeatures = [ "benchmark" "kvm" ]; + + +itchy cannot do builds that require +kvm, but scratchy does support +such builds. For regular builds, itchy will be +preferred over scratchy because it has a higher +speed factor. + +Remote builders can also be configured in +nix.conf, e.g. + + +builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd + + +Finally, remote builders can be configured in a separate configuration +file included in via the syntax +@file. For example, + + +builders = @/etc/nix/machines + + +causes the list of machines in /etc/nix/machines +to be included. (This is the default.) + +If you want the builders to use caches, you likely want to set +the option builders-use-substitutes +in your local nix.conf. + +To build only on remote builders and disable building on the local machine, +you can use the option . + + diff --git a/third_party/nix/doc/manual/advanced-topics/post-build-hook.xml b/third_party/nix/doc/manual/advanced-topics/post-build-hook.xml new file mode 100644 index 000000000..3dc43ee79 --- /dev/null +++ b/third_party/nix/doc/manual/advanced-topics/post-build-hook.xml @@ -0,0 +1,160 @@ + + +Using the <xref linkend="conf-post-build-hook" /> +Uploading to an S3-compatible binary cache after each build + + +
+ Implementation Caveats + Here we use the post-build hook to upload to a binary cache. + This is a simple and working example, but it is not suitable for all + use cases. + + The post build hook program runs after each executed build, + and blocks the build loop. The build loop exits if the hook program + fails. + + Concretely, this implementation will make Nix slow or unusable + when the internet is slow or unreliable. + + A more advanced implementation might pass the store paths to a + user-supplied daemon or queue for processing the store paths outside + of the build loop. +
+ +
+ Prerequisites + + + This tutorial assumes you have configured an S3-compatible binary cache + according to the instructions at + , and + that the root user's default AWS profile can + upload to the bucket. + +
+ +
+ Set up a Signing Key + Use nix-store --generate-binary-cache-key to + create our public and private signing keys. We will sign paths + with the private key, and distribute the public key for verifying + the authenticity of the paths. + + +# nix-store --generate-binary-cache-key example-nix-cache-1 /etc/nix/key.private /etc/nix/key.public +# cat /etc/nix/key.public +example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +Then, add the public key and the cache URL to your +nix.conf's +and like: + + +substituters = https://cache.nixos.org/ s3://example-nix-cache +trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= example-nix-cache-1:1/cKDz3QCCOmwcztD2eV6Coggp6rqc9DGjWv7C0G+rM= + + +we will restart the Nix daemon a later step. +
+ +
+ Implementing the build hook + Write the following script to + /etc/nix/upload-to-cache.sh: + + + +#!/bin/sh + +set -eu +set -f # disable globbing +export IFS=' ' + +echo "Signing paths" $OUT_PATHS +nix sign-paths --key-file /etc/nix/key.private $OUT_PATHS +echo "Uploading paths" $OUT_PATHS +exec nix copy --to 's3://example-nix-cache' $OUT_PATHS + + + + Should <literal>$OUT_PATHS</literal> be quoted? + + The $OUT_PATHS variable is a space-separated + list of Nix store paths. In this case, we expect and want the + shell to perform word splitting to make each output path its + own argument to nix sign-paths. Nix guarantees + the paths will not contain any spaces, however a store path + might contain glob characters. The set -f + disables globbing in the shell. + + + + Then make sure the hook program is executable by the root user: + +# chmod +x /etc/nix/upload-to-cache.sh + +
+ +
+ Updating Nix Configuration + + Edit /etc/nix/nix.conf to run our hook, + by adding the following configuration snippet at the end: + + +post-build-hook = /etc/nix/upload-to-cache.sh + + +Then, restart the nix-daemon. +
+ +
+ Testing + + Build any derivation, for example: + + +$ nix-build -E '(import <nixpkgs> {}).writeText "example" (builtins.toString builtins.currentTime)' +these derivations will be built: + /nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv +building '/nix/store/s4pnfbkalzy5qz57qs6yybna8wylkig6-example.drv'... +running post-build-hook '/home/grahamc/projects/github.com/NixOS/nix/post-hook.sh'... +post-build-hook: Signing paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +post-build-hook: Uploading paths /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +/nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + + Then delete the path from the store, and try substituting it from the binary cache: + +$ rm ./result +$ nix-store --delete /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example + + +Now, copy the path back from the cache: + +$ nix store --realize /nix/store/ibcyipq5gf91838ldx40mjsp0b8w9n18-example +copying path '/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example from 's3://example-nix-cache'... +warning: you did not specify '--add-root'; the result might be removed by the garbage collector +/nix/store/m8bmqwrch6l3h8s0k3d673xpmipcdpsa-example + +
+
+ Conclusion + + We now have a Nix installation configured to automatically sign and + upload every local build to a remote binary cache. + + + + Before deploying this to production, be sure to consider the + implementation caveats in . + +
+ diff --git a/third_party/nix/doc/manual/command-ref/command-ref.xml b/third_party/nix/doc/manual/command-ref/command-ref.xml new file mode 100644 index 000000000..cfad9b7d7 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/command-ref.xml @@ -0,0 +1,20 @@ + + +Command Reference + + +This section lists commands and options that you can use when you +work with Nix. + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/conf-file.xml b/third_party/nix/doc/manual/command-ref/conf-file.xml new file mode 100644 index 000000000..48dce7c95 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/conf-file.xml @@ -0,0 +1,1233 @@ + + + + + nix.conf + 5 + Nix + + + + + nix.conf + Nix configuration file + + +Description + +Nix reads settings from two configuration files: + + + + + The system-wide configuration file + sysconfdir/nix/nix.conf + (i.e. /etc/nix/nix.conf on most systems), or + $NIX_CONF_DIR/nix.conf if + NIX_CONF_DIR is set. + + + + The user configuration file + $XDG_CONFIG_HOME/nix/nix.conf, or + ~/.config/nix/nix.conf if + XDG_CONFIG_HOME is not set. + + + + +The configuration files consist of +name = +value pairs, one per line. Other +files can be included with a line like include +path, where +path is interpreted relative to the current +conf file and a missing file is an error unless +!include is used instead. +Comments start with a # character. Here is an +example configuration file: + + +keep-outputs = true # Nice for developers +keep-derivations = true # Idem + + +You can override settings on the command line using the + flag, e.g. --option keep-outputs +false. + +The following settings are currently available: + + + + + allowed-uris + + + + A list of URI prefixes to which access is allowed in + restricted evaluation mode. For example, when set to + https://github.com/NixOS, builtin functions + such as fetchGit are allowed to access + https://github.com/NixOS/patchelf.git. + + + + + + + allow-import-from-derivation + + By default, Nix allows you to import from a derivation, + allowing building at evaluation time. With this option set to false, Nix will throw an error + when evaluating an expression that uses this feature, allowing users to ensure their evaluation + will not require any builds to take place. + + + + + allow-new-privileges + + (Linux-specific.) By default, builders on Linux + cannot acquire new privileges by calling setuid/setgid programs or + programs that have file capabilities. For example, programs such + as sudo or ping will + fail. (Note that in sandbox builds, no such programs are available + unless you bind-mount them into the sandbox via the + option.) You can allow the + use of such programs by enabling this option. This is impure and + usually undesirable, but may be useful in certain scenarios + (e.g. to spin up containers or set up userspace network interfaces + in tests). + + + + + allowed-users + + + + A list of names of users (separated by whitespace) that + are allowed to connect to the Nix daemon. As with the + option, you can specify groups by + prefixing them with @. Also, you can allow + all users by specifying *. The default is + *. + + Note that trusted users are always allowed to connect. + + + + + + + auto-optimise-store + + If set to true, Nix + automatically detects files in the store that have identical + contents, and replaces them with hard links to a single copy. + This saves disk space. If set to false (the + default), you can still run nix-store + --optimise to get rid of duplicate + files. + + + + + builders + + A list of machines on which to perform builds. See for details. + + + + + builders-use-substitutes + + If set to true, Nix will instruct + remote build machines to use their own binary substitutes if available. In + practical terms, this means that remote hosts will fetch as many build + dependencies as possible from their own substitutes (e.g, from + cache.nixos.org), instead of waiting for this host to + upload them all. This can drastically reduce build times if the network + connection between this computer and the remote build host is slow. Defaults + to false. + + + + build-users-group + + This options specifies the Unix group containing + the Nix build user accounts. In multi-user Nix installations, + builds should not be performed by the Nix account since that would + allow users to arbitrarily modify the Nix store and database by + supplying specially crafted builders; and they cannot be performed + by the calling user since that would allow him/her to influence + the build result. + + Therefore, if this option is non-empty and specifies a valid + group, builds will be performed under the user accounts that are a + member of the group specified here (as listed in + /etc/group). Those user accounts should not + be used for any other purpose! + + Nix will never run two builds under the same user account at + the same time. This is to prevent an obvious security hole: a + malicious user writing a Nix expression that modifies the build + result of a legitimate Nix expression being built by another user. + Therefore it is good to have as many Nix build user accounts as + you can spare. (Remember: uids are cheap.) + + The build users should have permission to create files in + the Nix store, but not delete them. Therefore, + /nix/store should be owned by the Nix + account, its group should be the group specified here, and its + mode should be 1775. + + If the build users group is empty, builds will be performed + under the uid of the Nix process (that is, the uid of the caller + if NIX_REMOTE is empty, the uid under which the Nix + daemon runs if NIX_REMOTE is + daemon). Obviously, this should not be used in + multi-user settings with untrusted users. + + + + + + + compress-build-log + + If set to true (the default), + build logs written to /nix/var/log/nix/drvs + will be compressed on the fly using bzip2. Otherwise, they will + not be compressed. + + + + connect-timeout + + + + The timeout (in seconds) for establishing connections in + the binary cache substituter. It corresponds to + curl’s + option. + + + + + + + cores + + Sets the value of the + NIX_BUILD_CORES environment variable in the + invocation of builders. Builders can use this variable at their + discretion to control the maximum amount of parallelism. For + instance, in Nixpkgs, if the derivation attribute + enableParallelBuilding is set to + true, the builder passes the + flag to GNU Make. + It can be overridden using the command line switch and + defaults to 1. The value 0 + means that the builder should use all available CPU cores in the + system. + + See also . + + + diff-hook + + + Absolute path to an executable capable of diffing build results. + The hook executes if is + true, and the output of a build is known to not be the same. + This program is not executed to determine if two results are the + same. + + + + The diff hook is executed by the same user and group who ran the + build. However, the diff hook does not have write access to the + store path just built. + + + The diff hook program receives three parameters: + + + + + A path to the previous build's results + + + + + + A path to the current build's results + + + + + + The path to the build's derivation + + + + + + The path to the build's scratch directory. This directory + will exist only if the build was run with + . + + + + + + The stderr and stdout output from the diff hook will not be + displayed to the user. Instead, it will print to the nix-daemon's + log. + + + When using the Nix daemon, diff-hook must + be set in the nix.conf configuration file, and + cannot be passed at the command line. + + + + + + enforce-determinism + + See . + + + + extra-sandbox-paths + + A list of additional paths appended to + . Useful if you want to extend + its default value. + + + + + extra-platforms + + Platforms other than the native one which + this machine is capable of building for. This can be useful for + supporting additional architectures on compatible machines: + i686-linux can be built on x86_64-linux machines (and the default + for this setting reflects this); armv7 is backwards-compatible with + armv6 and armv5tel; some aarch64 machines can also natively run + 32-bit ARM code; and qemu-user may be used to support non-native + platforms (though this may be slow and buggy). Most values for this + are not enabled by default because build systems will often + misdetect the target platform and generate incompatible code, so you + may wish to cross-check the results of using this option against + proper natively-built versions of your + derivations. + + + + + extra-substituters + + Additional binary caches appended to those + specified in . When used by + unprivileged users, untrusted substituters (i.e. those not listed + in ) are silently + ignored. + + + + fallback + + If set to true, Nix will fall + back to building from source if a binary substitute fails. This + is equivalent to the flag. The + default is false. + + + + fsync-metadata + + If set to true, changes to the + Nix store metadata (in /nix/var/nix/db) are + synchronously flushed to disk. This improves robustness in case + of system crashes, but reduces performance. The default is + true. + + + + hashed-mirrors + + A list of web servers used by + builtins.fetchurl to obtain files by + hash. The default is + http://tarballs.nixos.org/. Given a hash type + ht and a base-16 hash + h, Nix will try to download the file + from + hashed-mirror/ht/h. + This allows files to be downloaded even if they have disappeared + from their original URI. For example, given the default mirror + http://tarballs.nixos.org/, when building the derivation + + +builtins.fetchurl { + url = https://example.org/foo-1.2.3.tar.xz; + sha256 = "2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae"; +} + + + Nix will attempt to download this file from + http://tarballs.nixos.org/sha256/2c26b46b68ffc68ff99b453c1d30413413422d706483bfa0f98a5e886266e7ae + first. If it is not available there, if will try the original URI. + + + + + http-connections + + The maximum number of parallel TCP connections + used to fetch files from binary caches and by other downloads. It + defaults to 25. 0 means no limit. + + + + + keep-build-log + + If set to true (the default), + Nix will write the build log of a derivation (i.e. the standard + output and error of its builder) to the directory + /nix/var/log/nix/drvs. The build log can be + retrieved using the command nix-store -l + path. + + + + + keep-derivations + + If true (default), the garbage + collector will keep the derivations from which non-garbage store + paths were built. If false, they will be + deleted unless explicitly registered as a root (or reachable from + other roots). + + Keeping derivation around is useful for querying and + traceability (e.g., it allows you to ask with what dependencies or + options a store path was built), so by default this option is on. + Turn it off to save a bit of disk space (or a lot if + keep-outputs is also turned on). + + + keep-env-derivations + + If false (default), derivations + are not stored in Nix user environments. That is, the derivations of + any build-time-only dependencies may be garbage-collected. + + If true, when you add a Nix derivation to + a user environment, the path of the derivation is stored in the + user environment. Thus, the derivation will not be + garbage-collected until the user environment generation is deleted + (nix-env --delete-generations). To prevent + build-time-only dependencies from being collected, you should also + turn on keep-outputs. + + The difference between this option and + keep-derivations is that this one is + “sticky”: it applies to any user environment created while this + option was enabled, while keep-derivations + only applies at the moment the garbage collector is + run. + + + + keep-outputs + + If true, the garbage collector + will keep the outputs of non-garbage derivations. If + false (default), outputs will be deleted unless + they are GC roots themselves (or reachable from other roots). + + In general, outputs must be registered as roots separately. + However, even if the output of a derivation is registered as a + root, the collector will still delete store paths that are used + only at build time (e.g., the C compiler, or source tarballs + downloaded from the network). To prevent it from doing so, set + this option to true. + + + max-build-log-size + + + + This option defines the maximum number of bytes that a + builder can write to its stdout/stderr. If the builder exceeds + this limit, it’s killed. A value of 0 (the + default) means that there is no limit. + + + + + + max-free + + When a garbage collection is triggered by the + min-free option, it stops as soon as + max-free bytes are available. The default is + infinity (i.e. delete all garbage). + + + + max-jobs + + This option defines the maximum number of jobs + that Nix will try to build in parallel. The default is + 1. The special value auto + causes Nix to use the number of CPUs in your system. 0 + is useful when using remote builders to prevent any local builds (except for + preferLocalBuild derivation attribute which executes locally + regardless). It can be + overridden using the () + command line switch. + + See also . + + + + max-silent-time + + + + This option defines the maximum number of seconds that a + builder can go without producing any data on standard output or + standard error. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite + loop, or to catch remote builds that are hanging due to network + problems. It can be overridden using the command + line switch. + + The value 0 means that there is no + timeout. This is also the default. + + + + + + min-free + + + When free disk space in /nix/store + drops below min-free during a build, Nix + performs a garbage-collection until max-free + bytes are available or there is no more garbage. A value of + 0 (the default) disables this feature. + + + + + narinfo-cache-negative-ttl + + + + The TTL in seconds for negative lookups. If a store path is + queried from a substituter but was not found, there will be a + negative lookup cached in the local disk cache database for the + specified duration. + + + + + + narinfo-cache-positive-ttl + + + + The TTL in seconds for positive lookups. If a store path is + queried from a substituter, the result of the query will be cached + in the local disk cache database including some of the NAR + metadata. The default TTL is a month, setting a shorter TTL for + positive lookups can be useful for binary caches that have + frequent garbage collection, in which case having a more frequent + cache invalidation would prevent trying to pull the path again and + failing with a hash mismatch if the build isn't reproducible. + + + + + + + netrc-file + + If set to an absolute path to a netrc + file, Nix will use the HTTP authentication credentials in this file when + trying to download from a remote host through HTTP or HTTPS. Defaults to + $NIX_CONF_DIR/netrc. + + The netrc file consists of a list of + accounts in the following format: + + +machine my-machine +login my-username +password my-password + + + For the exact syntax, see the + curl documentation. + + This must be an absolute path, and ~ + is not resolved. For example, ~/.netrc won't + resolve to your home directory's .netrc. + + + + + + + plugin-files + + + A list of plugin files to be loaded by Nix. Each of these + files will be dlopened by Nix, allowing them to affect + execution through static initialization. In particular, these + plugins may construct static instances of RegisterPrimOp to + add new primops or constants to the expression language, + RegisterStoreImplementation to add new store implementations, + RegisterCommand to add new subcommands to the + nix command, and RegisterSetting to add new + nix config settings. See the constructors for those types for + more details. + + + Since these files are loaded into the same address space as + Nix itself, they must be DSOs compatible with the instance of + Nix running at the time (i.e. compiled against the same + headers, not linked to any incompatible libraries). They + should not be linked to any Nix libs directly, as those will + be available already at load time. + + + If an entry in the list is a directory, all files in the + directory are loaded as plugins (non-recursively). + + + + + + pre-build-hook + + + + + If set, the path to a program that can set extra + derivation-specific settings for this system. This is used for settings + that can't be captured by the derivation model itself and are too variable + between different versions of the same system to be hard-coded into nix. + + + The hook is passed the derivation path and, if sandboxes are enabled, + the sandbox directory. It can then modify the sandbox and send a series of + commands to modify various settings to stdout. The currently recognized + commands are: + + + + extra-sandbox-paths + + + + Pass a list of files and directories to be included in the + sandbox for this build. One entry per line, terminated by an empty + line. Entries have the same format as + sandbox-paths. + + + + + + + + + + + post-build-hook + + Optional. The path to a program to execute after each build. + + This option is only settable in the global + nix.conf, or on the command line by trusted + users. + + When using the nix-daemon, the daemon executes the hook as + root. If the nix-daemon is not involved, the + hook runs as the user executing the nix-build. + + + The hook executes after an evaluation-time build. + The hook does not execute on substituted paths. + The hook's output always goes to the user's terminal. + If the hook fails, the build succeeds but no further builds execute. + The hook executes synchronously, and blocks other builds from progressing while it runs. + + + The program executes with no arguments. The program's environment + contains the following environment variables: + + + + DRV_PATH + + The derivation for the built paths. + Example: + /nix/store/5nihn1a7pa8b25l9zafqaqibznlvvp3f-bash-4.4-p23.drv + + + + + + OUT_PATHS + + Output paths of the built derivation, separated by a space character. + Example: + /nix/store/zf5lbh336mnzf1nlswdn11g4n2m8zh3g-bash-4.4-p23-dev + /nix/store/rjxwxwv1fpn9wa2x5ssk5phzwlcv4mna-bash-4.4-p23-doc + /nix/store/6bqvbzjkcp9695dq0dpl5y43nvy37pq1-bash-4.4-p23-info + /nix/store/r7fng3kk3vlpdlh2idnrbn37vh4imlj2-bash-4.4-p23-man + /nix/store/xfghy8ixrhz3kyy6p724iv3cxji088dx-bash-4.4-p23. + + + + + + See for an example + implementation. + + + + + repeat + + How many times to repeat builds to check whether + they are deterministic. The default value is 0. If the value is + non-zero, every build is repeated the specified number of + times. If the contents of any of the runs differs from the + previous ones and is + true, the build is rejected and the resulting store paths are not + registered as “valid” in Nix’s database. + + + require-sigs + + If set to true (the default), + any non-content-addressed path added or copied to the Nix store + (e.g. when substituting from a binary cache) must have a valid + signature, that is, be signed using one of the keys listed in + or + . Set to false + to disable signature checking. + + + + + restrict-eval + + + + If set to true, the Nix evaluator will + not allow access to any files outside of the Nix search path (as + set via the NIX_PATH environment variable or the + option), or to URIs outside of + . The default is + false. + + + + + + run-diff-hook + + + If true, enable the execution of . + + + + When using the Nix daemon, run-diff-hook must + be set in the nix.conf configuration file, + and cannot be passed at the command line. + + + + + sandbox + + If set to true, builds will be + performed in a sandboxed environment, i.e., + they’re isolated from the normal file system hierarchy and will + only see their dependencies in the Nix store, the temporary build + directory, private versions of /proc, + /dev, /dev/shm and + /dev/pts (on Linux), and the paths configured with the + sandbox-paths + option. This is useful to prevent undeclared dependencies + on files in directories such as /usr/bin. In + addition, on Linux, builds run in private PID, mount, network, IPC + and UTS namespaces to isolate them from other processes in the + system (except that fixed-output derivations do not run in private + network namespace to ensure they can access the network). + + Currently, sandboxing only work on Linux and macOS. The use + of a sandbox requires that Nix is run as root (so you should use + the “build users” + feature to perform the actual builds under different users + than root). + + If this option is set to relaxed, then + fixed-output derivations and derivations that have the + __noChroot attribute set to + true do not run in sandboxes. + + The default is true on Linux and + false on all other platforms. + + + + + + sandbox-dev-shm-size + + This option determines the maximum size of the + tmpfs filesystem mounted on + /dev/shm in Linux sandboxes. For the format, + see the description of the option of + tmpfs in + mount8. The + default is 50%. + + + + + + sandbox-paths + + A list of paths bind-mounted into Nix sandbox + environments. You can use the syntax + target=source + to mount a path in a different location in the sandbox; for + instance, /bin=/nix-bin will mount the path + /nix-bin as /bin inside the + sandbox. If source is followed by + ?, then it is not an error if + source does not exist; for example, + /dev/nvidiactl? specifies that + /dev/nvidiactl will only be mounted in the + sandbox if it exists in the host filesystem. + + Depending on how Nix was built, the default value for this option + may be empty or provide /bin/sh as a + bind-mount of bash. + + + + + secret-key-files + + A whitespace-separated list of files containing + secret (private) keys. These are used to sign locally-built + paths. They can be generated using nix-store + --generate-binary-cache-key. The corresponding public + key can be distributed to other users, who can add it to + in their + nix.conf. + + + + + show-trace + + Causes Nix to print out a stack trace in case of Nix + expression evaluation errors. + + + + + substitute + + If set to true (default), Nix + will use binary substitutes if available. This option can be + disabled to force building from source. + + + + stalled-download-timeout + + The timeout (in seconds) for receiving data from servers + during download. Nix cancels idle downloads after this timeout's + duration. + + + + substituters + + A list of URLs of substituters, separated by + whitespace. The default is + https://cache.nixos.org. + + + + system + + This option specifies the canonical Nix system + name of the current installation, such as + i686-linux or + x86_64-darwin. Nix can only build derivations + whose system attribute equals the value + specified here. In general, it never makes sense to modify this + value from its default, since you can use it to ‘lie’ about the + platform you are building on (e.g., perform a Mac OS build on a + Linux machine; the result would obviously be wrong). It only + makes sense if the Nix binaries can run on multiple platforms, + e.g., ‘universal binaries’ that run on x86_64-linux and + i686-linux. + + It defaults to the canonical Nix system name detected by + configure at build time. + + + + + system-features + + A set of system “features” supported by this + machine, e.g. kvm. Derivations can express a + dependency on such features through the derivation attribute + requiredSystemFeatures. For example, the + attribute + + +requiredSystemFeatures = [ "kvm" ]; + + + ensures that the derivation can only be built on a machine with + the kvm feature. + + This setting by default includes kvm if + /dev/kvm is accessible, and the + pseudo-features nixos-test, + benchmark and big-parallel + that are used in Nixpkgs to route builds to specific + machines. + + + + + + tarball-ttl + + + Default: 3600 seconds. + + The number of seconds a downloaded tarball is considered + fresh. If the cached tarball is stale, Nix will check whether + it is still up to date using the ETag header. Nix will download + a new version if the ETag header is unsupported, or the + cached ETag doesn't match. + + + Setting the TTL to 0 forces Nix to always + check if the tarball is up to date. + + Nix caches tarballs in + $XDG_CACHE_HOME/nix/tarballs. + + Files fetched via NIX_PATH, + fetchGit, fetchMercurial, + fetchTarball, and fetchurl + respect this TTL. + + + + + timeout + + + + This option defines the maximum number of seconds that a + builder can run. This is useful (for instance in an automated + build system) to catch builds that are stuck in an infinite loop + but keep writing to their standard output or standard error. It + can be overridden using the command line + switch. + + The value 0 means that there is no + timeout. This is also the default. + + + + + + trace-function-calls + + + + Default: false. + + If set to true, the Nix evaluator will + trace every function call. Nix will print a log message at the + "vomit" level for every function entrance and function exit. + + +function-trace entered undefined position at 1565795816999559622 +function-trace exited undefined position at 1565795816999581277 +function-trace entered /nix/store/.../example.nix:226:41 at 1565795253249935150 +function-trace exited /nix/store/.../example.nix:226:41 at 1565795253249941684 + + + The undefined position means the function + call is a builtin. + + Use the contrib/stack-collapse.py script + distributed with the Nix source code to convert the trace logs + in to a format suitable for flamegraph.pl. + + + + + + trusted-public-keys + + A whitespace-separated list of public keys. When + paths are copied from another Nix store (such as a binary cache), + they must be signed with one of these keys. For example: + cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + hydra.nixos.org-1:CNHJZBh9K4tP3EKF6FkkgeVYsS3ohTl+oS0Qa8bezVs=. + + + + trusted-substituters + + A list of URLs of substituters, separated by + whitespace. These are not used by default, but can be enabled by + users of the Nix daemon by specifying --option + substituters urls on the + command line. Unprivileged users are only allowed to pass a + subset of the URLs listed in substituters and + trusted-substituters. + + + + trusted-users + + + + A list of names of users (separated by whitespace) that + have additional rights when connecting to the Nix daemon, such + as the ability to specify additional binary caches, or to import + unsigned NARs. You can also specify groups by prefixing them + with @; for instance, + @wheel means all users in the + wheel group. The default is + root. + + Adding a user to + is essentially equivalent to giving that user root access to the + system. For example, the user can set + and thereby obtain read access to + directories that are otherwise inacessible to + them. + + + + + + + + + + Deprecated Settings + + + + + + + binary-caches + + Deprecated: + binary-caches is now an alias to + . + + + + binary-cache-public-keys + + Deprecated: + binary-cache-public-keys is now an alias to + . + + + + build-compress-log + + Deprecated: + build-compress-log is now an alias to + . + + + + build-cores + + Deprecated: + build-cores is now an alias to + . + + + + build-extra-chroot-dirs + + Deprecated: + build-extra-chroot-dirs is now an alias to + . + + + + build-extra-sandbox-paths + + Deprecated: + build-extra-sandbox-paths is now an alias to + . + + + + build-fallback + + Deprecated: + build-fallback is now an alias to + . + + + + build-max-jobs + + Deprecated: + build-max-jobs is now an alias to + . + + + + build-max-log-size + + Deprecated: + build-max-log-size is now an alias to + . + + + + build-max-silent-time + + Deprecated: + build-max-silent-time is now an alias to + . + + + + build-repeat + + Deprecated: + build-repeat is now an alias to + . + + + + build-timeout + + Deprecated: + build-timeout is now an alias to + . + + + + build-use-chroot + + Deprecated: + build-use-chroot is now an alias to + . + + + + build-use-sandbox + + Deprecated: + build-use-sandbox is now an alias to + . + + + + build-use-substitutes + + Deprecated: + build-use-substitutes is now an alias to + . + + + + gc-keep-derivations + + Deprecated: + gc-keep-derivations is now an alias to + . + + + + gc-keep-outputs + + Deprecated: + gc-keep-outputs is now an alias to + . + + + + env-keep-derivations + + Deprecated: + env-keep-derivations is now an alias to + . + + + + extra-binary-caches + + Deprecated: + extra-binary-caches is now an alias to + . + + + + trusted-binary-caches + + Deprecated: + trusted-binary-caches is now an alias to + . + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/env-common.xml b/third_party/nix/doc/manual/command-ref/env-common.xml new file mode 100644 index 000000000..696d68c34 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/env-common.xml @@ -0,0 +1,202 @@ + + +Common Environment Variables + + +Most Nix commands interpret the following environment variables: + + + +IN_NIX_SHELL + + Indicator that tells if the current environment was set up by + nix-shell. Since Nix 2.0 the values are + "pure" and "impure" + + + +NIX_PATH + + + + A colon-separated list of directories used to look up Nix + expressions enclosed in angle brackets (i.e., + <path>). For + instance, the value + + +/home/eelco/Dev:/etc/nixos + + will cause Nix to look for paths relative to + /home/eelco/Dev and + /etc/nixos, in that order. It is also + possible to match paths against a prefix. For example, the value + + +nixpkgs=/home/eelco/Dev/nixpkgs-branch:/etc/nixos + + will cause Nix to search for + <nixpkgs/path> in + /home/eelco/Dev/nixpkgs-branch/path + and + /etc/nixos/nixpkgs/path. + + If a path in the Nix search path starts with + http:// or https://, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must consist of a + single top-level directory. For example, setting + NIX_PATH to + + +nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-15.09.tar.gz + + tells Nix to download the latest revision in the Nixpkgs/NixOS + 15.09 channel. + + A following shorthand can be used to refer to the official channels: + + nixpkgs=channel:nixos-15.09 + + + The search path can be extended using the option, which takes precedence over + NIX_PATH. + + + + +NIX_IGNORE_SYMLINK_STORE + + + + Normally, the Nix store directory (typically + /nix/store) is not allowed to contain any + symlink components. This is to prevent “impure” builds. Builders + sometimes “canonicalise” paths by resolving all symlink components. + Thus, builds on different machines (with + /nix/store resolving to different locations) + could yield different results. This is generally not a problem, + except when builds are deployed to machines where + /nix/store resolves differently. If you are + sure that you’re not going to do that, you can set + NIX_IGNORE_SYMLINK_STORE to 1. + + Note that if you’re symlinking the Nix store so that you can + put it on another file system than the root file system, on Linux + you’re better off using bind mount points, e.g., + + +$ mkdir /nix +$ mount -o bind /mnt/otherdisk/nix /nix + + Consult the mount + 8 manual page for details. + + + + + + +NIX_STORE_DIR + + Overrides the location of the Nix store (default + prefix/store). + + + + +NIX_DATA_DIR + + Overrides the location of the Nix static data + directory (default + prefix/share). + + + + +NIX_LOG_DIR + + Overrides the location of the Nix log directory + (default prefix/var/log/nix). + + + + +NIX_STATE_DIR + + Overrides the location of the Nix state directory + (default prefix/var/nix). + + + + +NIX_CONF_DIR + + Overrides the location of the Nix configuration + directory (default + prefix/etc/nix). + + + + +TMPDIR + + Use the specified directory to store temporary + files. In particular, this includes temporary build directories; + these can take up substantial amounts of disk space. The default is + /tmp. + + + + +NIX_REMOTE + + This variable should be set to + daemon if you want to use the Nix daemon to + execute Nix operations. This is necessary in multi-user Nix installations. + If the Nix daemon's Unix socket is at some non-standard path, + this variable should be set to unix://path/to/socket. + Otherwise, it should be left unset. + + + + +NIX_SHOW_STATS + + If set to 1, Nix will print some + evaluation statistics, such as the number of values + allocated. + + + + +NIX_COUNT_CALLS + + If set to 1, Nix will print how + often functions were called during Nix expression evaluation. This + is useful for profiling your Nix expressions. + + + + +GC_INITIAL_HEAP_SIZE + + If Nix has been configured to use the Boehm garbage + collector, this variable sets the initial size of the heap in bytes. + It defaults to 384 MiB. Setting it to a low value reduces memory + consumption, but will increase runtime due to the overhead of + garbage collection. + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/files.xml b/third_party/nix/doc/manual/command-ref/files.xml new file mode 100644 index 000000000..7bbc96e89 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/files.xml @@ -0,0 +1,14 @@ + + +Files + +This section lists configuration files that you can use when you +work with Nix. + + + + \ No newline at end of file diff --git a/third_party/nix/doc/manual/command-ref/main-commands.xml b/third_party/nix/doc/manual/command-ref/main-commands.xml new file mode 100644 index 000000000..0f4169243 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/main-commands.xml @@ -0,0 +1,17 @@ + + +Main Commands + +This section lists commands and options that you can use when you +work with Nix. + + + + + + + \ No newline at end of file diff --git a/third_party/nix/doc/manual/command-ref/nix-build.xml b/third_party/nix/doc/manual/command-ref/nix-build.xml new file mode 100644 index 000000000..c1b783c87 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-build.xml @@ -0,0 +1,190 @@ + + + + nix-build + 1 + Nix + + + + + nix-build + build a Nix expression + + + + + nix-build + + name value + name value + + + + + + attrPath + + + + + + + + + outlink + + paths + + + +Description + +The nix-build command builds the derivations +described by the Nix expressions in paths. +If the build succeeds, it places a symlink to the result in the +current directory. The symlink is called result. +If there are multiple Nix expressions, or the Nix expressions evaluate +to multiple derivations, multiple sequentially numbered symlinks are +created (result, result-2, +and so on). + +If no paths are specified, then +nix-build will use default.nix +in the current directory, if it exists. + +If an element of paths starts with +http:// or https://, it is +interpreted as the URL of a tarball that will be downloaded and +unpacked to a temporary location. The tarball must include a single +top-level directory containing at least a file named +default.nix. + +nix-build is essentially a wrapper around +nix-instantiate +(to translate a high-level Nix expression to a low-level store +derivation) and nix-store +--realise (to build the store derivation). + +The result of the build is automatically registered as +a root of the Nix garbage collector. This root disappears +automatically when the result symlink is deleted +or renamed. So don’t rename the symlink. + + + + +Options + +All options not listed here are passed to nix-store +--realise, except for and + / which are passed to +nix-instantiate. See +also . + + + + + + Do not create a symlink to the output path. Note + that as a result the output does not become a root of the garbage + collector, and so might be deleted by nix-store + --gc. + + + + + Show what store paths would be built or downloaded + + + / + outlink + + Change the name of the symlink to the output path + created from result to + outlink. + + + + + +The following common options are supported: + + + + + + + + +Examples + + +$ nix-build '<nixpkgs>' -A firefox +store derivation is /nix/store/qybprl8sz2lc...-firefox-1.5.0.7.drv +/nix/store/d18hyl92g30l...-firefox-1.5.0.7 + +$ ls -l result +lrwxrwxrwx ... result -> /nix/store/d18hyl92g30l...-firefox-1.5.0.7 + +$ ls ./result/bin/ +firefox firefox-config + +If a derivation has multiple outputs, +nix-build will build the default (first) output. +You can also build all outputs: + +$ nix-build '<nixpkgs>' -A openssl.all + +This will create a symlink for each output named +result-outputname. +The suffix is omitted if the output name is out. +So if openssl has outputs out, +bin and man, +nix-build will create symlinks +result, result-bin and +result-man. It’s also possible to build a specific +output: + +$ nix-build '<nixpkgs>' -A openssl.man + +This will create a symlink result-man. + +Build a Nix expression given on the command line: + + +$ nix-build -E 'with import <nixpkgs> { }; runCommand "foo" { } "echo bar > $out"' +$ cat ./result +bar + + + + +Build the GNU Hello package from the latest revision of the +master branch of Nixpkgs: + + +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello + + + + + + + +Environment variables + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-channel.xml b/third_party/nix/doc/manual/command-ref/nix-channel.xml new file mode 100644 index 000000000..5a2866e6b --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-channel.xml @@ -0,0 +1,178 @@ + + + + nix-channel + 1 + Nix + + + + + nix-channel + manage Nix channels + + + + + nix-channel + + url name + name + + names + generation + + + + +Description + +A Nix channel is a mechanism that allows you to automatically +stay up-to-date with a set of pre-built Nix expressions. A Nix +channel is just a URL that points to a place containing a set of Nix +expressions. See also . + +This command has the following operations: + + + + url [name] + + Adds a channel named + name with URL + url to the list of subscribed channels. + If name is omitted, it defaults to the + last component of url, with the + suffixes -stable or + -unstable removed. + + + + name + + Removes the channel named + name from the list of subscribed + channels. + + + + + + Prints the names and URLs of all subscribed + channels on standard output. + + + + [names…] + + Downloads the Nix expressions of all subscribed + channels (or only those included in + names if specified) and makes them the + default for nix-env operations (by symlinking + them from the directory + ~/.nix-defexpr). + + + + [generation] + + Reverts the previous call to nix-channel + --update. Optionally, you can specify a specific channel + generation number to restore. + + + + + + + +Note that does not automatically perform +an update. + +The list of subscribed channels is stored in +~/.nix-channels. + + + +Examples + +To subscribe to the Nixpkgs channel and install the GNU Hello package: + + +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --update +$ nix-env -iA nixpkgs.hello + +You can revert channel updates using : + + +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +"14.04.527.0e935f1" + +$ nix-channel --rollback +switching from generation 483 to 482 + +$ nix-instantiate --eval -E '(import <nixpkgs> {}).lib.nixpkgsVersion' +"14.04.526.dbadfad" + + + + +Files + + + + /nix/var/nix/profiles/per-user/username/channels + + nix-channel uses a + nix-env profile to keep track of previous + versions of the subscribed channels. Every time you run + nix-channel --update, a new channel generation + (that is, a symlink to the channel Nix expressions in the Nix store) + is created. This enables nix-channel --rollback + to revert to previous versions. + + + + ~/.nix-defexpr/channels + + This is a symlink to + /nix/var/nix/profiles/per-user/username/channels. It + ensures that nix-env can find your channels. In + a multi-user installation, you may also have + ~/.nix-defexpr/channels_root, which links to + the channels of the root user. + + + + + + + +Channel format + +A channel URL should point to a directory containing the +following files: + + + + nixexprs.tar.xz + + A tarball containing Nix expressions and files + referenced by them (such as build scripts and patches). At the + top level, the tarball should contain a single directory. That + directory must contain a file default.nix + that serves as the channel’s “entry point”. + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-collect-garbage.xml b/third_party/nix/doc/manual/command-ref/nix-collect-garbage.xml new file mode 100644 index 000000000..43e068796 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-collect-garbage.xml @@ -0,0 +1,63 @@ + + + + nix-collect-garbage + 1 + Nix + + + + + nix-collect-garbage + delete unreachable store paths + + + + + nix-collect-garbage + + + period + bytes + + + + +Description + +The command nix-collect-garbage is mostly an +alias of nix-store +--gc, that is, it deletes all unreachable paths in +the Nix store to clean up your system. However, it provides two +additional options: (), +which deletes all old generations of all profiles in +/nix/var/nix/profiles by invoking +nix-env --delete-generations old on all profiles +(of course, this makes rollbacks to previous configurations +impossible); and + period, +where period is a value such as 30d, which deletes +all generations older than the specified number of days in all profiles +in /nix/var/nix/profiles (except for the generations +that were active at that point in time). + + + + +Example + +To delete from the Nix store everything that is not used by the +current generations of each profile, do + + +$ nix-collect-garbage -d + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-copy-closure.xml b/third_party/nix/doc/manual/command-ref/nix-copy-closure.xml new file mode 100644 index 000000000..e6dcf180a --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-copy-closure.xml @@ -0,0 +1,169 @@ + + + + nix-copy-closure + 1 + Nix + + + + + nix-copy-closure + copy a closure to or from a remote machine via SSH + + + + + nix-copy-closure + + + + + + + + + + + + + + user@machine + + paths + + + + +Description + +nix-copy-closure gives you an easy and +efficient way to exchange software between machines. Given one or +more Nix store paths on the local +machine, nix-copy-closure computes the closure of +those paths (i.e. all their dependencies in the Nix store), and copies +all paths in the closure to the remote machine via the +ssh (Secure Shell) command. With the +, the direction is reversed: +the closure of paths on a remote machine is +copied to the Nix store on the local machine. + +This command is efficient because it only sends the store paths +that are missing on the target machine. + +Since nix-copy-closure calls +ssh, you may be asked to type in the appropriate +password or passphrase. In fact, you may be asked +twice because nix-copy-closure +currently connects twice to the remote machine, first to get the set +of paths missing on the target machine, and second to send the dump of +those paths. If this bothers you, use +ssh-agent. + + +Options + + + + + + Copy the closure of + paths from the local Nix store to the + Nix store on machine. This is the + default. + + + + + + Copy the closure of + paths from the Nix store on + machine to the local Nix + store. + + + + + + Enable compression of the SSH + connection. + + + + + + Also copy the outputs of store derivations + included in the closure. + + + + / + + Attempt to download missing paths on the target + machine using Nix’s substitute mechanism. Any paths that cannot + be substituted on the target are still copied normally from the + source. This is useful, for instance, if the connection between + the source and target machine is slow, but the connection between + the target machine and nixos.org (the default + binary cache server) is fast. + + + + + + Show verbose output. + + + + + + + + +Environment variables + + + + NIX_SSHOPTS + + Additional options to be passed to + ssh on the command line. + + + + + + + + +Examples + +Copy Firefox with all its dependencies to a remote machine: + + +$ nix-copy-closure --to alice@itchy.labs $(type -tP firefox) + + + +Copy Subversion from a remote machine and then install it into a +user environment: + + +$ nix-copy-closure --from alice@itchy.labs \ + /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 +$ nix-env -i /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 + + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-daemon.xml b/third_party/nix/doc/manual/command-ref/nix-daemon.xml new file mode 100644 index 000000000..a2161f033 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-daemon.xml @@ -0,0 +1,35 @@ + + + + nix-daemon + 8 + Nix + + + + + nix-daemon + Nix multi-user support daemon + + + + + nix-daemon + + + + +Description + +The Nix daemon is necessary in multi-user Nix installations. It +performs build actions and other operations on the Nix store on behalf +of unprivileged users. + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-env.xml b/third_party/nix/doc/manual/command-ref/nix-env.xml new file mode 100644 index 000000000..d257a5e49 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-env.xml @@ -0,0 +1,1505 @@ + + + + nix-env + 1 + Nix + + + + + nix-env + manipulate or query Nix user environments + + + + + nix-env + + name value + name value + + + + + + path + + + + + + + path + + + + system + + + operation + options + arguments + + + + +Description + +The command nix-env is used to manipulate Nix +user environments. User environments are sets of software packages +available to a user at some point in time. In other words, they are a +synthesised view of the programs available in the Nix store. There +may be many user environments: different users can have different +environments, and individual users can switch between different +environments. + +nix-env takes exactly one +operation flag which indicates the subcommand to +be performed. These are documented below. + + + + + + + +Selectors + +Several commands, such as nix-env -q and +nix-env -i, take a list of arguments that specify +the packages on which to operate. These are extended regular +expressions that must match the entire name of the package. (For +details on regular expressions, see +regex7.) +The match is case-sensitive. The regular expression can optionally be +followed by a dash and a version number; if omitted, any version of +the package will match. Here are some examples: + + + + + firefox + Matches the package name + firefox and any version. + + + + firefox-32.0 + Matches the package name + firefox and version + 32.0. + + + + gtk\\+ + Matches the package name + gtk+. The + character must + be escaped using a backslash to prevent it from being interpreted + as a quantifier, and the backslash must be escaped in turn with + another backslash to ensure that the shell passes it + on. + + + + .\* + Matches any package name. This is the default for + most commands. + + + + '.*zip.*' + Matches any package name containing the string + zip. Note the dots: '*zip*' + does not work, because in a regular expression, the character + * is interpreted as a + quantifier. + + + + '.*(firefox|chromium).*' + Matches any package name containing the strings + firefox or + chromium. + + + + + + + + + + + + +Common options + +This section lists the options that are common to all +operations. These options are allowed for every subcommand, though +they may not always have an effect. See +also . + + + + / path + + Specifies the Nix expression (designated below as + the active Nix expression) used by the + , , and + operations to obtain + derivations. The default is + ~/.nix-defexpr. + + If the argument starts with http:// or + https://, it is interpreted as the URL of a + tarball that will be downloaded and unpacked to a temporary + location. The tarball must include a single top-level directory + containing at least a file named default.nix. + + + + + + / path + + Specifies the profile to be used by those + operations that operate on a profile (designated below as the + active profile). A profile is a sequence of + user environments called generations, one of + which is the current + generation. + + + + + + For the , + , , + , + and + operations, this flag will cause + nix-env to print what + would be done if this flag had not been + specified, without actually doing it. + + also prints out which paths will + be substituted (i.e., + downloaded) and which paths will be built from source (because no + substitute is available). + + + + system + + By default, operations such as show derivations matching any platform. This + option allows you to use derivations for the specified platform + system. + + + + + + + + + + + + + + + +Files + + + + ~/.nix-defexpr + + The source for the default Nix + expressions used by the , + , and operations to obtain derivations. The + option may be used to override this + default. + + If ~/.nix-defexpr is a file, + it is loaded as a Nix expression. If the expression + is a set, it is used as the default Nix expression. + If the expression is a function, an empty set is passed + as argument and the return value is used as + the default Nix expression. + + If ~/.nix-defexpr is a directory + containing a default.nix file, that file + is loaded as in the above paragraph. + + If ~/.nix-defexpr is a directory without + a default.nix file, then its contents + (both files and subdirectories) are loaded as Nix expressions. + The expressions are combined into a single set, each expression + under an attribute with the same name as the original file + or subdirectory. + + + For example, if ~/.nix-defexpr contains + two files, foo.nix and bar.nix, + then the default Nix expression will essentially be + + +{ + foo = import ~/.nix-defexpr/foo.nix; + bar = import ~/.nix-defexpr/bar.nix; +} + + + + The file manifest.nix is always ignored. + Subdirectories without a default.nix file + are traversed recursively in search of more Nix expressions, + but the names of these intermediate directories are not + added to the attribute paths of the default Nix expression. + + The command nix-channel places symlinks + to the downloaded Nix expressions from each subscribed channel in + this directory. + + + + + ~/.nix-profile + + A symbolic link to the user's current profile. By + default, this symlink points to + prefix/var/nix/profiles/default. + The PATH environment variable should include + ~/.nix-profile/bin for the user environment + to be visible to the user. + + + + + + + + + + + +Operation <option>--install</option> + +Synopsis + + + nix-env + + + + + + + + + + + + + + args + + + + + +Description + +The install operation creates a new user environment, based on +the current generation of the active profile, to which a set of store +paths described by args is added. The +arguments args map to store paths in a +number of possible ways: + + + + By default, args is a set + of derivation names denoting derivations in the active Nix + expression. These are realised, and the resulting output paths are + installed. Currently installed derivations with a name equal to the + name of a derivation being added are removed unless the option + is + specified. + + If there are multiple derivations matching a name in + args that have the same name (e.g., + gcc-3.3.6 and gcc-4.1.1), then + the derivation with the highest priority is + used. A derivation can define a priority by declaring the + meta.priority attribute. This attribute should + be a number, with a higher value denoting a lower priority. The + default priority is 0. + + If there are multiple matching derivations with the same + priority, then the derivation with the highest version will be + installed. + + You can force the installation of multiple derivations with + the same name by being specific about the versions. For instance, + nix-env -i gcc-3.3.6 gcc-4.1.1 will install both + version of GCC (and will probably cause a user environment + conflict!). + + If + () is specified, the arguments are + attribute paths that select attributes from the + top-level Nix expression. This is faster than using derivation + names and unambiguous. To find out the attribute paths of available + packages, use nix-env -qaP. + + If + path is given, + args is a set of names denoting installed + store paths in the profile path. This is + an easy way to copy user environment elements from one profile to + another. + + If is given, + args are Nix functions that are called with the + active Nix expression as their single argument. The derivations + returned by those function calls are installed. This allows + derivations to be specified in an unambiguous way, which is necessary + if there are multiple derivations with the same + name. + + If args are store + derivations, then these are realised, and the resulting + output paths are installed. + + If args are store paths + that are not store derivations, then these are realised and + installed. + + By default all outputs are installed for each derivation. + That can be reduced by setting meta.outputsToInstall. + + + + + + + + + +Flags + + + + / + + Use only derivations for which a substitute is + registered, i.e., there is a pre-built binary available that can + be downloaded in lieu of building the derivation. Thus, no + packages will be built from source. + + + + + + + Do not remove derivations with a name matching one + of the derivations being installed. Usually, trying to have two + versions of the same package installed in the same generation of a + profile will lead to an error in building the generation, due to + file name clashes between the two versions. However, this is not + the case for all packages. + + + + + + + Remove all previously installed packages first. + This is equivalent to running nix-env -e '.*' + first, except that everything happens in a single + transaction. + + + + + + + + +Examples + +To install a specific version of gcc from the +active Nix expression: + + +$ nix-env --install gcc-3.3.2 +installing `gcc-3.3.2' +uninstalling `gcc-3.1' + +Note the previously installed version is removed, since + was not specified. + +To install an arbitrary version: + + +$ nix-env --install gcc +installing `gcc-3.3.2' + + + +To install using a specific attribute: + + +$ nix-env -i -A gcc40mips +$ nix-env -i -A xorg.xorgserver + + + +To install all derivations in the Nix expression foo.nix: + + +$ nix-env -f ~/foo.nix -i '.*' + + + +To copy the store path with symbolic name gcc +from another profile: + + +$ nix-env -i --from-profile /nix/var/nix/profiles/foo gcc + + + +To install a specific store derivation (typically created by +nix-instantiate): + + +$ nix-env -i /nix/store/fibjb1bfbpm5mrsxc4mh2d8n37sxh91i-gcc-3.4.3.drv + + + +To install a specific output path: + + +$ nix-env -i /nix/store/y3cgx0xj1p4iv9x0pnnmdhr8iyg741vk-gcc-3.4.3 + + + +To install from a Nix expression specified on the command-line: + + +$ nix-env -f ./foo.nix -i -E \ + 'f: (f {system = "i686-linux";}).subversionWithJava' + +I.e., this evaluates to (f: (f {system = +"i686-linux";}).subversionWithJava) (import ./foo.nix), thus +selecting the subversionWithJava attribute from the +set returned by calling the function defined in +./foo.nix. + +A dry-run tells you which paths will be downloaded or built from +source: + + +$ nix-env -f '<nixpkgs>' -iA hello --dry-run +(dry run; not doing anything) +installing ‘hello-2.10’ +these paths will be fetched (0.04 MiB download, 0.19 MiB unpacked): + /nix/store/wkhdf9jinag5750mqlax6z2zbwhqb76n-hello-2.10 + ... + + + +To install Firefox from the latest revision in the Nixpkgs/NixOS +14.12 channel: + + +$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox + + +(The GitHub repository nixpkgs-channels is updated +automatically from the main nixpkgs repository +after certain tests have succeeded and binaries have been built and +uploaded to the binary cache at cache.nixos.org.) + + + + + + + + + +Operation <option>--upgrade</option> + +Synopsis + + + nix-env + + + + + + + + + + + + args + + + + +Description + +The upgrade operation creates a new user environment, based on +the current generation of the active profile, in which all store paths +are replaced for which there are newer versions in the set of paths +described by args. Paths for which there +are no newer versions are left untouched; this is not an error. It is +also not an error if an element of args +matches no installed derivations. + +For a description of how args is +mapped to a set of store paths, see . If +args describes multiple store paths with +the same symbolic name, only the one with the highest version is +installed. + + + +Flags + + + + + + Only upgrade a derivation to newer versions. This + is the default. + + + + + + In addition to upgrading to newer versions, also + “upgrade” to derivations that have the same version. Version are + not a unique identification of a derivation, so there may be many + derivations that have the same version. This flag may be useful + to force “synchronisation” between the installed and available + derivations. + + + + + + Only “upgrade” to derivations + that have the same version. This may not seem very useful, but it + actually is, e.g., when there is a new release of Nixpkgs and you + want to replace installed applications with the same versions + built against newer dependencies (to reduce the number of + dependencies floating around on your system). + + + + + + In addition to upgrading to newer versions, also + “upgrade” to derivations that have the same or a lower version. + I.e., derivations may actually be downgraded depending on what is + available in the active Nix expression. + + + + + +For the other flags, see . + + + +Examples + + +$ nix-env --upgrade gcc +upgrading `gcc-3.3.1' to `gcc-3.4' + +$ nix-env -u gcc-3.3.2 --always (switch to a specific version) +upgrading `gcc-3.4' to `gcc-3.3.2' + +$ nix-env --upgrade pan +(no upgrades available, so nothing happens) + +$ nix-env -u (try to upgrade everything) +upgrading `hello-2.1.2' to `hello-2.1.3' +upgrading `mozilla-1.2' to `mozilla-1.4' + + + +Versions + +The upgrade operation determines whether a derivation +y is an upgrade of a derivation +x by looking at their respective +name attributes. The names (e.g., +gcc-3.3.1 are split into two parts: the package +name (gcc), and the version +(3.3.1). The version part starts after the first +dash not followed by a letter. x is considered an +upgrade of y if their package names match, and the +version of y is higher that that of +x. + +The versions are compared by splitting them into contiguous +components of numbers and letters. E.g., 3.3.1pre5 +is split into [3, 3, 1, "pre", 5]. These lists are +then compared lexicographically (from left to right). Corresponding +components a and b are compared +as follows. If they are both numbers, integer comparison is used. If +a is an empty string and b is a +number, a is considered less than +b. The special string component +pre (for pre-release) is +considered to be less than other components. String components are +considered less than number components. Otherwise, they are compared +lexicographically (i.e., using case-sensitive string comparison). + +This is illustrated by the following examples: + + +1.0 < 2.3 +2.1 < 2.3 +2.3 = 2.3 +2.5 > 2.3 +3.1 > 2.3 +2.3.1 > 2.3 +2.3.1 > 2.3a +2.3pre1 < 2.3 +2.3pre3 < 2.3pre12 +2.3a < 2.3c +2.3pre1 < 2.3c +2.3pre1 < 2.3q + + + + + + + + + + + +Operation <option>--uninstall</option> + +Synopsis + + + nix-env + + + + + drvnames + + + +Description + +The uninstall operation creates a new user environment, based on +the current generation of the active profile, from which the store +paths designated by the symbolic names +names are removed. + + + +Examples + + +$ nix-env --uninstall gcc +$ nix-env -e '.*' (remove everything) + + + + + + + + + +Operation <option>--set</option> + +Synopsis + + + nix-env + + drvname + + + +Description + +The operation modifies the current generation of a +profile so that it contains exactly the specified derivation, and nothing else. + + + + +Examples + + +The following updates a profile such that its current generation will contain +just Firefox: + + +$ nix-env -p /nix/var/nix/profiles/browser --set firefox + + + + + + + + + + + +Operation <option>--set-flag</option> + +Synopsis + + + nix-env + + name + value + drvnames + + + +Description + +The operation allows meta attributes +of installed packages to be modified. There are several attributes +that can be usefully modified, because they affect the behaviour of +nix-env or the user environment build +script: + + + + priority can be changed to + resolve filename clashes. The user environment build script uses + the meta.priority attribute of derivations to + resolve filename collisions between packages. Lower priority values + denote a higher priority. For instance, the GCC wrapper package and + the Binutils package in Nixpkgs both have a file + bin/ld, so previously if you tried to install + both you would get a collision. Now, on the other hand, the GCC + wrapper declares a higher priority than Binutils, so the former’s + bin/ld is symlinked in the user + environment. + + keep can be set to + true to prevent the package from being upgraded + or replaced. This is useful if you want to hang on to an older + version of a package. + + active can be set to + false to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to true to re-enable the + package. + + + + + + + +Examples + +To prevent the currently installed Firefox from being upgraded: + + +$ nix-env --set-flag keep true firefox + +After this, nix-env -u will ignore Firefox. + +To disable the currently installed Firefox, then install a new +Firefox while the old remains part of the profile: + + +$ nix-env -q +firefox-2.0.0.9 (the current one) + +$ nix-env --preserve-installed -i firefox-2.0.0.11 +installing `firefox-2.0.0.11' +building path(s) `/nix/store/myy0y59q3ig70dgq37jqwg1j0rsapzsl-user-environment' +collision between `/nix/store/...-firefox-2.0.0.11/bin/firefox' + and `/nix/store/...-firefox-2.0.0.9/bin/firefox'. +(i.e., can’t have two active at the same time) + +$ nix-env --set-flag active false firefox +setting flag on `firefox-2.0.0.9' + +$ nix-env --preserve-installed -i firefox-2.0.0.11 +installing `firefox-2.0.0.11' + +$ nix-env -q +firefox-2.0.0.11 (the enabled one) +firefox-2.0.0.9 (the disabled one) + + + +To make files from binutils take precedence +over files from gcc: + + +$ nix-env --set-flag priority 5 binutils +$ nix-env --set-flag priority 10 gcc + + + + + + + + + + + +Operation <option>--query</option> + +Synopsis + + + nix-env + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + attribute-path + + + + + names + + + + + +Description + +The query operation displays information about either the store +paths that are installed in the current generation of the active +profile (), or the derivations that are +available for installation in the active Nix expression +(). It only prints information about +derivations whose symbolic name matches one of +names. + +The derivations are sorted by their name +attributes. + + + + +Source selection + +The following flags specify the set of things on which the query +operates. + + + + + + The query operates on the store paths that are + installed in the current generation of the active profile. This + is the default. + + + + + + + The query operates on the derivations that are + available in the active Nix expression. + + + + + + + + +Queries + +The following flags specify what information to display about +the selected derivations. Multiple flags may be specified, in which +case the information is shown in the order given here. Note that the +name of the derivation is shown unless is +specified. + + + + + + + + Print the result in an XML representation suitable + for automatic processing by other tools. The root element is + called items, which contains a + item element for each available or installed + derivation. The fields discussed below are all stored in + attributes of the item + elements. + + + + + + Print the result in a JSON representation suitable + for automatic processing by other tools. + + + + / + + Show only derivations for which a substitute is + registered, i.e., there is a pre-built binary available that can + be downloaded in lieu of building the derivation. Thus, this + shows all packages that probably can be installed + quickly. + + + + + + + Print the status of the + derivation. The status consists of three characters. The first + is I or -, indicating + whether the derivation is currently installed in the current + generation of the active profile. This is by definition the case + for , but not for + . The second is P + or -, indicating whether the derivation is + present on the system. This indicates whether installation of an + available derivation will require the derivation to be built. The + third is S or -, indicating + whether a substitute is available for the + derivation. + + + + + + + Print the attribute path of + the derivation, which can be used to unambiguously select it using + the option + available in commands that install derivations like + nix-env --install. + + + + + + Suppress printing of the name + attribute of each derivation. + + + + / + + + Compare installed versions to available versions, + or vice versa (if is given). This is + useful for quickly seeing whether upgrades for installed + packages are available in a Nix expression. A column is added + with the following meaning: + + + + < version + + A newer version of the package is available + or installed. + + + + = version + + At most the same version of the package is + available or installed. + + + + > version + + Only older versions of the package are + available or installed. + + + + - ? + + No version of the package is available or + installed. + + + + + + + + + + + + Print the system attribute of + the derivation. + + + + + + Print the path of the store + derivation. + + + + + + Print the output path of the + derivation. + + + + + + Print a short (one-line) description of the + derivation, if available. The description is taken from the + meta.description attribute of the + derivation. + + + + + + Print all of the meta-attributes of the + derivation. This option is only available with + or . + + + + + + + + +Examples + +To show installed packages: + + +$ nix-env -q +bison-1.875c +docbook-xml-4.2 +firefox-1.0.4 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… + + + + +To show available packages: + + +$ nix-env -qa +firefox-1.0.7 +GConf-2.4.0.1 +MPlayer-1.0pre7 +ORBit2-2.8.3 +… + + + + +To show the status of available packages: + + +$ nix-env -qas +-P- firefox-1.0.7 (not installed but present) +--S GConf-2.4.0.1 (not present, but there is a substitute for fast installation) +--S MPlayer-1.0pre3 (i.e., this is not the installed MPlayer, even though the version is the same!) +IP- ORBit2-2.8.3 (installed and by definition present) +… + + + + +To show available packages in the Nix expression foo.nix: + + +$ nix-env -f ./foo.nix -qa +foo-1.2.3 + + + + +To compare installed versions to what’s available: + + +$ nix-env -qc +... +acrobat-reader-7.0 - ? (package is not available at all) +autoconf-2.59 = 2.59 (same version) +firefox-1.0.4 < 1.0.7 (a more recent version is available) +... + + + + +To show all packages with “zip” in the name: + + +$ nix-env -qa '.*zip.*' +bzip2-1.0.6 +gzip-1.6 +zip-3.0 +… + + + + +To show all packages with “firefox” or +“chromium” in the name: + + +$ nix-env -qa '.*(firefox|chromium).*' +chromium-37.0.2062.94 +chromium-beta-38.0.2125.24 +firefox-32.0.3 +firefox-with-plugins-13.0.1 +… + + + + +To show all packages in the latest revision of the Nixpkgs +repository: + + +$ nix-env -f https://github.com/NixOS/nixpkgs/archive/master.tar.gz -qa + + + + + + + + + + + + +Operation <option>--switch-profile</option> + +Synopsis + + + nix-env + + + + + path + + + + + +Description + +This operation makes path the current +profile for the user. That is, the symlink +~/.nix-profile is made to point to +path. + + + +Examples + + +$ nix-env -S ~/my-profile + + + + + + + + + +Operation <option>--list-generations</option> + +Synopsis + + + nix-env + + + + + + +Description + +This operation print a list of all the currently existing +generations for the active profile. These may be switched to using +the operation. It also prints +the creation date of the generation, and indicates the current +generation. + + + + +Examples + + +$ nix-env --list-generations + 95 2004-02-06 11:48:24 + 96 2004-02-06 11:49:01 + 97 2004-02-06 16:22:45 + 98 2004-02-06 16:24:33 (current) + + + + + + + + + +Operation <option>--delete-generations</option> + +Synopsis + + + nix-env + + generations + + + + + +Description + +This operation deletes the specified generations of the current +profile. The generations can be a list of generation numbers, the +special value old to delete all non-current +generations, a value such as 30d to delete all +generations older than the specified number of days (except for the +generation that was active at that point in time), or a value such as ++5 to keep the last 5 generations +ignoring any newer than current, e.g., if 30 is the current +generation +5 will delete generation 25 +and all older generations. +Periodically deleting old generations is important to make garbage collection +effective. + + + +Examples + + +$ nix-env --delete-generations 3 4 8 + +$ nix-env --delete-generations +5 + +$ nix-env --delete-generations 30d + +$ nix-env -p other_profile --delete-generations old + + + + + + + + + +Operation <option>--switch-generation</option> + +Synopsis + + + nix-env + + + + + generation + + + + + +Description + +This operation makes generation number +generation the current generation of the +active profile. That is, if the +profile is the path to +the active profile, then the symlink +profile is made to +point to +profile-generation-link, +which is in turn a symlink to the actual user environment in the Nix +store. + +Switching will fail if the specified generation does not exist. + + + + +Examples + + +$ nix-env -G 42 +switching from generation 50 to 42 + + + + + + + + + +Operation <option>--rollback</option> + +Synopsis + + + nix-env + + + + + +Description + +This operation switches to the “previous” generation of the +active profile, that is, the highest numbered generation lower than +the current generation, if it exists. It is just a convenience +wrapper around and +. + + + + +Examples + + +$ nix-env --rollback +switching from generation 92 to 91 + +$ nix-env --rollback +error: no generation older than the current (91) exists + + + + + + +Environment variables + + + + NIX_PROFILE + + Location of the Nix profile. Defaults to the + target of the symlink ~/.nix-profile, if it + exists, or /nix/var/nix/profiles/default + otherwise. + + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-hash.xml b/third_party/nix/doc/manual/command-ref/nix-hash.xml new file mode 100644 index 000000000..80263e18e --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-hash.xml @@ -0,0 +1,176 @@ + + + + nix-hash + 1 + Nix + + + + + nix-hash + compute the cryptographic hash of a path + + + + + nix-hash + + + + hashAlgo + path + + + nix-hash + + hash + + + nix-hash + + hash + + + + +Description + +The command nix-hash computes the +cryptographic hash of the contents of each +path and prints it on standard output. By +default, it computes an MD5 hash, but other hash algorithms are +available as well. The hash is printed in hexadecimal. To generate +the same hash as nix-prefetch-url you have to +specify multiple arguments, see below for an example. + +The hash is computed over a serialisation +of each path: a dump of the file system tree rooted at the path. This +allows directories and symlinks to be hashed as well as regular files. +The dump is in the NAR format produced by nix-store +. Thus, nix-hash +path yields the same +cryptographic hash as nix-store --dump +path | md5sum. + + + + +Options + + + + + + Print the cryptographic hash of the contents of + each regular file path. That is, do + not compute the hash over the dump of + path. The result is identical to that + produced by the GNU commands md5sum and + sha1sum. + + + + + + Print the hash in a base-32 representation rather + than hexadecimal. This base-32 representation is more compact and + can be used in Nix expressions (such as in calls to + fetchurl). + + + + + + Truncate hashes longer than 160 bits (such as + SHA-256) to 160 bits. + + + + hashAlgo + + Use the specified cryptographic hash algorithm, + which can be one of md5, + sha1, and + sha256. + + + + + + Don’t hash anything, but convert the base-32 hash + representation hash to + hexadecimal. + + + + + + Don’t hash anything, but convert the hexadecimal + hash representation hash to + base-32. + + + + + + + + +Examples + +Computing the same hash as nix-prefetch-url: + +$ nix-prefetch-url file://<(echo test) +1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj +$ nix-hash --type sha256 --flat --base32 <(echo test) +1lkgqb6fclns49861dwk9rzb6xnfkxbpws74mxnx01z9qyv1pjpj + + + +Computing hashes: + + +$ mkdir test +$ echo "hello" > test/world + +$ nix-hash test/ (MD5 hash; default) +8179d3caeff1869b5ba1744e5a245c04 + +$ nix-store --dump test/ | md5sum (for comparison) +8179d3caeff1869b5ba1744e5a245c04 - + +$ nix-hash --type sha1 test/ +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + +$ nix-hash --type sha1 --base32 test/ +nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 + +$ nix-hash --type sha256 --flat test/ +error: reading file `test/': Is a directory + +$ nix-hash --type sha256 --flat test/world +5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 + + + +Converting between hexadecimal and base-32: + + +$ nix-hash --type sha1 --to-base32 e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 +nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 + +$ nix-hash --type sha1 --to-base16 nvd61k9nalji1zl9rrdfmsmvyyjqpzg4 +e4fd8ba5f7bbeaea5ace89fe10255536cd60dab6 + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-instantiate.xml b/third_party/nix/doc/manual/command-ref/nix-instantiate.xml new file mode 100644 index 000000000..53f06aed1 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-instantiate.xml @@ -0,0 +1,266 @@ + + + + nix-instantiate + 1 + Nix + + + + + nix-instantiate + instantiate store derivations from Nix expressions + + + + + nix-instantiate + + + + + + + + + + + name value + + + + + + attrPath + + path + + + + + + files + + + nix-instantiate + + files + + + + +Description + +The command nix-instantiate generates store derivations from (high-level) +Nix expressions. It evaluates the Nix expressions in each of +files (which defaults to +./default.nix). Each top-level expression +should evaluate to a derivation, a list of derivations, or a set of +derivations. The paths of the resulting store derivations are printed +on standard output. + +If files is the character +-, then a Nix expression will be read from standard +input. + +See also for a list of common options. + + + + +Options + + + + + path + + + See the corresponding + options in nix-store. + + + + + + Just parse the input files, and print their + abstract syntax trees on standard output in ATerm + format. + + + + + + Just parse and evaluate the input files, and print + the resulting values on standard output. No instantiation of + store derivations takes place. + + + + + + Look up the given files in Nix’s search path (as + specified by the NIX_PATH + environment variable). If found, print the corresponding absolute + paths on standard output. For instance, if + NIX_PATH is + nixpkgs=/home/alice/nixpkgs, then + nix-instantiate --find-file nixpkgs/default.nix + will print + /home/alice/nixpkgs/default.nix. + + + + + + When used with , + recursively evaluate list elements and attributes. Normally, such + sub-expressions are left unevaluated (since the Nix expression + language is lazy). + + This option can cause non-termination, because lazy + data structures can be infinitely large. + + + + + + + + When used with , print the resulting + value as an JSON representation of the abstract syntax tree rather + than as an ATerm. + + + + + + When used with , print the resulting + value as an XML representation of the abstract syntax tree rather than as + an ATerm. The schema is the same as that used by the toXML built-in. + + + + + + + When used with , perform + evaluation in read/write mode so nix language features that + require it will still work (at the cost of needing to do + instantiation of every evaluated derivation). If this option is + not enabled, there may be uninstantiated store paths in the final + output. + + + + + + + + + + + + + + +Examples + +Instantiating store derivations from a Nix expression, and +building them using nix-store: + + +$ nix-instantiate test.nix (instantiate) +/nix/store/cigxbmvy6dzix98dxxh9b6shg7ar5bvs-perl-BerkeleyDB-0.26.drv + +$ nix-store -r $(nix-instantiate test.nix) (build) +... +/nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 (output path) + +$ ls -l /nix/store/qhqk4n8ci095g3sdp93x7rgwyh9rdvgk-perl-BerkeleyDB-0.26 +dr-xr-xr-x 2 eelco users 4096 1970-01-01 01:00 lib +... + + + +You can also give a Nix expression on the command line: + + +$ nix-instantiate -E 'with import <nixpkgs> { }; hello' +/nix/store/j8s4zyv75a724q38cb0r87rlczaiag4y-hello-2.8.drv + + +This is equivalent to: + + +$ nix-instantiate '<nixpkgs>' -A hello + + + + +Parsing and evaluating Nix expressions: + + +$ nix-instantiate --parse -E '1 + 2' +1 + 2 + +$ nix-instantiate --eval -E '1 + 2' +3 + +$ nix-instantiate --eval --xml -E '1 + 2' + + + +]]> + + + +The difference between non-strict and strict evaluation: + + +$ nix-instantiate --eval --xml -E 'rec { x = "foo"; y = x; }' +... + + + + + ]]> +... + +Note that y is left unevaluated (the XML +representation doesn’t attempt to show non-normal forms). + + +$ nix-instantiate --eval --xml --strict -E 'rec { x = "foo"; y = x; }' +... + + + + + ]]> +... + + + + + + +Environment variables + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-prefetch-url.xml b/third_party/nix/doc/manual/command-ref/nix-prefetch-url.xml new file mode 100644 index 000000000..621ded72e --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-prefetch-url.xml @@ -0,0 +1,131 @@ + + + + nix-prefetch-url + 1 + Nix + + + + + nix-prefetch-url + copy a file from a URL into the store and print its hash + + + + + nix-prefetch-url + + hashAlgo + + + name + url + hash + + + +Description + +The command nix-prefetch-url downloads the +file referenced by the URL url, prints its +cryptographic hash, and copies it into the Nix store. The file name +in the store is +hash-baseName, +where baseName is everything following the +final slash in url. + +This command is just a convenience for Nix expression writers. +Often a Nix expression fetches some source distribution from the +network using the fetchurl expression contained in +Nixpkgs. However, fetchurl requires a +cryptographic hash. If you don't know the hash, you would have to +download the file first, and then fetchurl would +download it again when you build your Nix expression. Since +fetchurl uses the same name for the downloaded file +as nix-prefetch-url, the redundant download can be +avoided. + +If hash is specified, then a download +is not performed if the Nix store already contains a file with the +same hash and base name. Otherwise, the file is downloaded, and an +error is signaled if the actual hash of the file does not match the +specified hash. + +This command prints the hash on standard output. Additionally, +if the option is used, the path of the +downloaded file in the Nix store is also printed. + + + + +Options + + + + hashAlgo + + Use the specified cryptographic hash algorithm, + which can be one of md5, + sha1, and + sha256. + + + + + + Print the store path of the downloaded file on + standard output. + + + + + + Unpack the archive (which must be a tarball or zip + file) and add the result to the Nix store. The resulting hash can + be used with functions such as Nixpkgs’s + fetchzip or + fetchFromGitHub. + + + + name + + Override the name of the file in the Nix store. By + default, this is + hash-basename, + where basename is the last component of + url. Overriding the name is necessary + when basename contains characters that + are not allowed in Nix store paths. + + + + + + + + +Examples + + +$ nix-prefetch-url ftp://ftp.gnu.org/pub/gnu/hello/hello-2.10.tar.gz +0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i + +$ nix-prefetch-url --print-path mirror://gnu/hello/hello-2.10.tar.gz +0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + +$ nix-prefetch-url --unpack --print-path https://github.com/NixOS/patchelf/archive/0.8.tar.gz +079agjlv0hrv7fxnx9ngipx14gyncbkllxrp9cccnh3a50fxcmy7 +/nix/store/19zrmhm3m40xxaw81c8cqm6aljgrnwj2-0.8.tar.gz + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-shell.xml b/third_party/nix/doc/manual/command-ref/nix-shell.xml new file mode 100644 index 000000000..bb4a4e420 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-shell.xml @@ -0,0 +1,397 @@ + + + + nix-shell + 1 + Nix + + + + + nix-shell + start an interactive shell based on a Nix expression + + + + + nix-shell + name value + name value + + + + + + attrPath + + cmd + cmd + regexp + + name + + + + + + + packages + + path + + + + +Description + +The command nix-shell will build the +dependencies of the specified derivation, but not the derivation +itself. It will then start an interactive shell in which all +environment variables defined by the derivation +path have been set to their corresponding +values, and the script $stdenv/setup has been +sourced. This is useful for reproducing the environment of a +derivation for development. + +If path is not given, +nix-shell defaults to +shell.nix if it exists, and +default.nix otherwise. + +If path starts with +http:// or https://, it is +interpreted as the URL of a tarball that will be downloaded and +unpacked to a temporary location. The tarball must include a single +top-level directory containing at least a file named +default.nix. + +If the derivation defines the variable +shellHook, it will be evaluated after +$stdenv/setup has been sourced. Since this hook is +not executed by regular Nix builds, it allows you to perform +initialisation specific to nix-shell. For example, +the derivation attribute + + +shellHook = + '' + echo "Hello shell" + ''; + + +will cause nix-shell to print Hello shell. + + + + +Options + +All options not listed here are passed to nix-store +--realise, except for and + / which are passed to +nix-instantiate. See +also . + + + + cmd + + In the environment of the derivation, run the + shell command cmd. This command is + executed in an interactive shell. (Use to + use a non-interactive shell instead.) However, a call to + exit is implicitly added to the command, so the + shell will exit after running the command. To prevent this, add + return at the end; e.g. --command + "echo Hello; return" will print Hello + and then drop you into the interactive shell. This can be useful + for doing any additional initialisation. + + + + cmd + + Like , but executes the + command in a non-interactive shell. This means (among other + things) that if you hit Ctrl-C while the command is running, the + shell exits. + + + + regexp + + Do not build any dependencies whose store path + matches the regular expression regexp. + This option may be specified multiple times. + + + + + + If this flag is specified, the environment is + almost entirely cleared before the interactive shell is started, + so you get an environment that more closely corresponds to the + “real” Nix build. A few variables, in particular + HOME, USER and + DISPLAY, are retained. Note that + ~/.bashrc and (depending on your Bash + installation) /etc/bashrc are still sourced, + so any variables set there will affect the interactive + shell. + + + + / packages… + + Set up an environment in which the specified + packages are present. The command line arguments are interpreted + as attribute names inside the Nix Packages collection. Thus, + nix-shell -p libjpeg openjdk will start a shell + in which the packages denoted by the attribute names + libjpeg and openjdk are + present. + + + + interpreter + + The chained script interpreter to be invoked by + nix-shell. Only applicable in + #!-scripts (described below). + + + + name + + When a shell is started, + keep the listed environment variables. + + + + + +The following common options are supported: + + + + + + + + +Environment variables + + + + NIX_BUILD_SHELL + + Shell used to start the interactive environment. + Defaults to the bash found in PATH. + + + + + + + + +Examples + +To build the dependencies of the package Pan, and start an +interactive shell in which to build it: + + +$ nix-shell '<nixpkgs>' -A pan +[nix-shell]$ unpackPhase +[nix-shell]$ cd pan-* +[nix-shell]$ configurePhase +[nix-shell]$ buildPhase +[nix-shell]$ ./pan/gui/pan + + +To clear the environment first, and do some additional automatic +initialisation of the interactive shell: + + +$ nix-shell '<nixpkgs>' -A pan --pure \ + --command 'export NIX_DEBUG=1; export NIX_CORES=8; return' + + +Nix expressions can also be given on the command line. For instance, +the following starts a shell containing the packages +sqlite and libX11: + + +$ nix-shell -E 'with import <nixpkgs> { }; runCommand "dummy" { buildInputs = [ sqlite xorg.libX11 ]; } ""' + + +A shorter way to do the same is: + + +$ nix-shell -p sqlite xorg.libX11 +[nix-shell]$ echo $NIX_LDFLAGS +… -L/nix/store/j1zg5v…-sqlite-3.8.0.2/lib -L/nix/store/0gmcz9…-libX11-1.6.1/lib … + + +The -p flag looks up Nixpkgs in the Nix search +path. You can override it by passing or setting +NIX_PATH. For example, the following gives you a shell +containing the Pan package from a specific revision of Nixpkgs: + + +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz + +[nix-shell:~]$ pan --version +Pan 0.139 + + + + + + + +Use as a <literal>#!</literal>-interpreter + +You can use nix-shell as a script interpreter +to allow scripts written in arbitrary languages to obtain their own +dependencies via Nix. This is done by starting the script with the +following lines: + + +#! /usr/bin/env nix-shell +#! nix-shell -i real-interpreter -p packages + + +where real-interpreter is the “real” script +interpreter that will be invoked by nix-shell after +it has obtained the dependencies and initialised the environment, and +packages are the attribute names of the +dependencies in Nixpkgs. + +The lines starting with #! nix-shell specify +nix-shell options (see above). Note that you cannot +write #! /usr/bin/env nix-shell -i ... because +many operating systems only allow one argument in +#! lines. + +For example, here is a Python script that depends on Python and +the prettytable package: + + +#! /usr/bin/env nix-shell +#! nix-shell -i python -p python pythonPackages.prettytable + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t + + + + +Similarly, the following is a Perl script that specifies that it +requires Perl and the HTML::TokeParser::Simple and +LWP packages: + + +#! /usr/bin/env nix-shell +#! nix-shell -i perl -p perl perlPackages.HTMLTokeParserSimple perlPackages.LWP + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} + + + + +Sometimes you need to pass a simple Nix expression to customize +a package like Terraform: + + + +You must use double quotes (") when +passing a simple Nix expression in a nix-shell shebang. + + +Finally, using the merging of multiple nix-shell shebangs the +following Haskell script uses a specific branch of Nixpkgs/NixOS (the +18.03 stable branch): + + + +If you want to be even more precise, you can specify a specific +revision of Nixpkgs: + + +#! nix-shell -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/0672315759b3e15e2121365f067c1c8c56bb4722.tar.gz + + + + +The examples above all used to get +dependencies from Nixpkgs. You can also use a Nix expression to build +your own dependencies. For example, the Python example could have been +written as: + + +#! /usr/bin/env nix-shell +#! nix-shell deps.nix -i python + + +where the file deps.nix in the same directory +as the #!-script contains: + + +with import <nixpkgs> {}; + +runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" + + + + + + + +Environment variables + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/nix-store.xml b/third_party/nix/doc/manual/command-ref/nix-store.xml new file mode 100644 index 000000000..113a3c2e4 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/nix-store.xml @@ -0,0 +1,1525 @@ + + + + nix-store + 1 + Nix + + + + + nix-store + manipulate or query the Nix store + + + + + nix-store + + path + + operation + options + arguments + + + + +Description + +The command nix-store performs primitive +operations on the Nix store. You generally do not need to run this +command manually. + +nix-store takes exactly one +operation flag which indicates the subcommand to +be performed. These are documented below. + + + + + + + +Common options + +This section lists the options that are common to all +operations. These options are allowed for every subcommand, though +they may not always have an effect. See +also for a list of common +options. + + + + path + + Causes the result of a realisation + ( and ) + to be registered as a root of the garbage collector (see ). The root is stored in + path, which must be inside a directory + that is scanned for roots by the garbage collector (i.e., + typically in a subdirectory of + /nix/var/nix/gcroots/) + unless the flag + is used. + + If there are multiple results, then multiple symlinks will + be created by sequentially numbering symlinks beyond the first one + (e.g., foo, foo-2, + foo-3, and so on). + + + + + + + + In conjunction with , this option + allows roots to be stored outside of the GC + roots directory. This is useful for commands such as + nix-build that place a symlink to the build + result in the current directory; such a build result should not be + garbage-collected unless the symlink is removed. + + The flag causes a uniquely named + symlink to path to be stored in + /nix/var/nix/gcroots/auto/. For instance, + + +$ nix-store --add-root /home/eelco/bla/result --indirect -r ... + +$ ls -l /nix/var/nix/gcroots/auto +lrwxrwxrwx 1 ... 2005-03-13 21:10 dn54lcypm8f8... -> /home/eelco/bla/result + +$ ls -l /home/eelco/bla/result +lrwxrwxrwx 1 ... 2005-03-13 21:10 /home/eelco/bla/result -> /nix/store/1r11343n6qd4...-f-spot-0.0.10 + + Thus, when /home/eelco/bla/result is removed, + the GC root in the auto directory becomes a + dangling symlink and will be ignored by the collector. + + Note that it is not possible to move or rename + indirect GC roots, since the symlink in the + auto directory will still point to the old + location. + + + + + + + + + + + + + + + + + +Operation <option>--realise</option> + +Synopsis + + + nix-store + + + + + paths + + + + + +Description + +The operation essentially “builds” +the specified store paths. Realisation is a somewhat overloaded term: + + + + If the store path is a + derivation, realisation ensures that the output + paths of the derivation are valid (i.e., the output path and its + closure exist in the file system). This can be done in several + ways. First, it is possible that the outputs are already valid, in + which case we are done immediately. Otherwise, there may be substitutes that produce the + outputs (e.g., by downloading them). Finally, the outputs can be + produced by performing the build action described by the + derivation. + + If the store path is not a derivation, realisation + ensures that the specified path is valid (i.e., it and its closure + exist in the file system). If the path is already valid, we are + done immediately. Otherwise, the path and any missing paths in its + closure may be produced through substitutes. If there are no + (successful) subsitutes, realisation fails. + + + + + +The output path of each derivation is printed on standard +output. (For non-derivations argument, the argument itself is +printed.) + +The following flags are available: + + + + + + Print on standard error a description of what + packages would be built or downloaded, without actually performing + the operation. + + + + + + If a non-derivation path does not have a + substitute, then silently ignore it. + + + + + + This option allows you to check whether a + derivation is deterministic. It rebuilds the specified derivation + and checks whether the result is bitwise-identical with the + existing outputs, printing an error if that’s not the case. The + outputs of the specified derivation must already exist. When used + with , if an output path is not identical to + the corresponding output from the previous build, the new output + path is left in + /nix/store/name.check. + + See also the configuration + option, which repeats a derivation a number of times and prevents + its outputs from being registered as “valid” in the Nix store + unless they are identical. + + + + + +Special exit codes: + + + + 100 + Generic build failure, the builder process + returned with a non-zero exit code. + + + 101 + Build timeout, the build was aborted because it + did not complete within the specified timeout. + + + + 102 + Hash mismatch, the build output was rejected + because it does not match the specified outputHash. + + + + 104 + Not deterministic, the build succeeded in check + mode but the resulting output is not binary reproducable. + + + + + +With the flag it's possible for +multiple failures to occur, in this case the 1xx status codes are or combined +using binary or. +1100100 + ^^^^ + |||`- timeout + ||`-- output hash mismatch + |`--- build failure + `---- not deterministic + + + + + +Examples + +This operation is typically used to build store derivations +produced by nix-instantiate: + + +$ nix-store -r $(nix-instantiate ./test.nix) +/nix/store/31axcgrlbfsxzmfff1gyj1bf62hvkby2-aterm-2.3.1 + +This is essentially what nix-build does. + +To test whether a previously-built derivation is deterministic: + + +$ nix-build '<nixpkgs>' -A hello --check -K + + + + + + + + + + + + + +Operation <option>--serve</option> + +Synopsis + + + nix-store + + + + + + +Description + +The operation provides access to +the Nix store over stdin and stdout, and is intended to be used +as a means of providing Nix store access to a restricted ssh user. + + +The following flags are available: + + + + + + Allow the connected client to request the realization + of derivations. In effect, this can be used to make the host act + as a remote builder. + + + + + + + + +Examples + +To turn a host into a build server, the +authorized_keys file can be used to provide build +access to a given SSH public key: + + +$ cat <<EOF >>/root/.ssh/authorized_keys +command="nice -n20 nix-store --serve --write" ssh-rsa AAAAB3NzaC1yc2EAAAA... +EOF + + + + + + + + + + + + + +Operation <option>--gc</option> + +Synopsis + + + nix-store + + + + + + + + bytes + + + + +Description + +Without additional flags, the operation +performs a garbage collection on the Nix store. That is, all paths in +the Nix store not reachable via file system references from a set of +“roots”, are deleted. + +The following suboperations may be specified: + + + + + + This operation prints on standard output the set + of roots used by the garbage collector. What constitutes a root + is described in . + + + + + + This operation prints on standard output the set + of “live” store paths, which are all the store paths reachable + from the roots. Live paths should never be deleted, since that + would break consistency — it would become possible that + applications are installed that reference things that are no + longer present in the store. + + + + + + This operation prints out on standard output the + set of “dead” store paths, which is just the opposite of the set + of live paths: any path in the store that is not live (with + respect to the roots) is dead. + + + + + + This operation performs an actual garbage + collection. All dead paths are removed from the + store. This is the default. + + + + + +By default, all unreachable paths are deleted. The following +options control what gets deleted and in what order: + + + + bytes + + Keep deleting paths until at least + bytes bytes have been deleted, then + stop. The argument bytes can be + followed by the multiplicative suffix K, + M, G or + T, denoting KiB, MiB, GiB or TiB + units. + + + + + + + +The behaviour of the collector is also influenced by the keep-outputs +and keep-derivations +variables in the Nix configuration file. + +With , the collector prints the total +number of freed bytes when it finishes (or when it is interrupted). +With , it prints the number of bytes that +would be freed. + + + + +Examples + +To delete all unreachable paths, just do: + + +$ nix-store --gc +deleting `/nix/store/kq82idx6g0nyzsp2s14gfsc38npai7lf-cairo-1.0.4.tar.gz.drv' +... +8825586 bytes freed (8.42 MiB) + + + +To delete at least 100 MiBs of unreachable paths: + + +$ nix-store --gc --max-freed $((100 * 1024 * 1024)) + + + + + + + + + + + + +Operation <option>--delete</option> + +Synopsis + + + nix-store + + + paths + + + + +Description + +The operation deletes the store paths +paths from the Nix store, but only if it is +safe to do so; that is, when the path is not reachable from a root of +the garbage collector. This means that you can only delete paths that +would also be deleted by nix-store --gc. Thus, +--delete is a more targeted version of +--gc. + +With the option , reachability +from the roots is ignored. However, the path still won’t be deleted +if there are other paths in the store that refer to it (i.e., depend +on it). + + + +Example + + +$ nix-store --delete /nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4 +0 bytes freed (0.00 MiB) +error: cannot delete path `/nix/store/zq0h41l75vlb4z45kzgjjmsjxvcv1qk7-mesa-6.4' since it is still alive + + + + + + + + + +Operation <option>--query</option> + +Synopsis + + + nix-store + + + + + + + + + + + + + + + + name + name + + + + + + + + + paths + + + + + +Description + +The operation displays various bits of +information about the store paths . The queries are described below. At +most one query can be specified. The default query is +. + +The paths paths may also be symlinks +from outside of the Nix store, to the Nix store. In that case, the +query is applied to the target of the symlink. + + + + + +Common query options + + + + + + + For each argument to the query that is a store + derivation, apply the query to the output path of the derivation + instead. + + + + + + + Realise each argument to the query first (see + nix-store + --realise). + + + + + + + + +Queries + + + + + + Prints out the output paths of the store + derivations paths. These are the paths + that will be produced when the derivation is + built. + + + + + + + Prints out the closure of the store path + paths. + + This query has one option: + + + + + + Also include the output path of store + derivations, and their closures. + + + + + + This query can be used to implement various kinds of + deployment. A source deployment is obtained + by distributing the closure of a store derivation. A + binary deployment is obtained by distributing + the closure of an output path. A cache + deployment (combined source/binary deployment, + including binaries of build-time-only dependencies) is obtained by + distributing the closure of a store derivation and specifying the + option . + + + + + + + + Prints the set of references of the store paths + paths, that is, their immediate + dependencies. (For all dependencies, use + .) + + + + + + Prints the set of referrers of + the store paths paths, that is, the + store paths currently existing in the Nix store that refer to one + of paths. Note that contrary to the + references, the set of referrers is not constant; it can change as + store paths are added or removed. + + + + + + Prints the closure of the set of store paths + paths under the referrers relation; that + is, all store paths that directly or indirectly refer to one of + paths. These are all the path currently + in the Nix store that are dependent on + paths. + + + + + + + Prints the deriver of the store paths + paths. If the path has no deriver + (e.g., if it is a source file), or if the deriver is not known + (e.g., in the case of a binary-only deployment), the string + unknown-deriver is printed. + + + + + + Prints the references graph of the store paths + paths in the format of the + dot tool of AT&T's Graphviz package. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path. + + + + + + Prints the references graph of the store paths + paths as a nested ASCII tree. + References are ordered by descending closure size; this tends to + flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the + graph. + + + + + + Prints the references graph of the store paths + paths in the GraphML file format. + This can be used to visualise dependency graphs. To obtain a + build-time dependency graph, apply this to a store derivation. To + obtain a runtime dependency graph, apply it to an output + path. + + + + name + name + + Prints the value of the attribute + name (i.e., environment variable) of + the store derivations paths. It is an + error for a derivation to not have the specified + attribute. + + + + + + Prints the SHA-256 hash of the contents of the + store paths paths (that is, the hash of + the output of nix-store --dump on the given + paths). Since the hash is stored in the Nix database, this is a + fast operation. + + + + + + Prints the size in bytes of the contents of the + store paths paths — to be precise, the + size of the output of nix-store --dump on the + given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + + + + + + Prints the garbage collector roots that point, + directly or indirectly, at the store paths + paths. + + + + + + + + +Examples + +Print the closure (runtime dependencies) of the +svn program in the current user environment: + + +$ nix-store -qR $(which svn) +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/9lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 +... + + + +Print the build-time dependencies of svn: + + +$ nix-store -qR $(nix-store -qd $(which svn)) +/nix/store/02iizgn86m42q905rddvg4ja975bk2i4-grep-2.5.1.tar.bz2.drv +/nix/store/07a2bzxmzwz5hp58nf03pahrv2ygwgs3-gcc-wrapper.sh +/nix/store/0ma7c9wsbaxahwwl04gbw3fcd806ski4-glibc-2.3.4.drv +... lots of other paths ... + +The difference with the previous example is that we ask the closure of +the derivation (), not the closure of the output +path that contains svn. + +Show the build-time dependencies as a tree: + + +$ nix-store -q --tree $(nix-store -qd $(which svn)) +/nix/store/7i5082kfb6yjbqdbiwdhhza0am2xvh6c-subversion-1.1.4.drv ++---/nix/store/d8afh10z72n8l1cr5w42366abiblgn54-builder.sh ++---/nix/store/fmzxmpjx2lh849ph0l36snfj9zdibw67-bash-3.0.drv +| +---/nix/store/570hmhmx3v57605cqg9yfvvyh0nnb8k8-bash +| +---/nix/store/p3srsbd8dx44v2pg6nbnszab5mcwx03v-builder.sh +... + + + +Show all paths that depend on the same OpenSSL library as +svn: + + +$ nix-store -q --referrers $(nix-store -q --binding openssl $(nix-store -qd $(which svn))) +/nix/store/23ny9l9wixx21632y2wi4p585qhva1q8-sylpheed-1.0.0 +/nix/store/5mbglq5ldqld8sj57273aljwkfvj22mc-subversion-1.1.4 +/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3 +/nix/store/l51240xqsgg8a7yrbqdx1rfzyv6l26fx-lynx-2.8.5 + + + +Show all paths that directly or indirectly depend on the Glibc +(C library) used by svn: + + +$ nix-store -q --referrers-closure $(ldd $(which svn) | grep /libc.so | awk '{print $3}') +/nix/store/034a6h4vpz9kds5r6kzb9lhh81mscw43-libgnomeprintui-2.8.2 +/nix/store/15l3yi0d45prm7a82pcrknxdh6nzmxza-gawk-3.1.4 +... + +Note that ldd is a command that prints out the +dynamic libraries used by an ELF executable. + +Make a picture of the runtime dependency graph of the current +user environment: + + +$ nix-store -q --graph ~/.nix-profile | dot -Tps > graph.ps +$ gv graph.ps + + + +Show every garbage collector root that points to a store path +that depends on svn: + + +$ nix-store -q --roots $(which svn) +/nix/var/nix/profiles/default-81-link +/nix/var/nix/profiles/default-82-link +/nix/var/nix/profiles/per-user/eelco/profile-97-link + + + + + + + + + + + + + + + + + + + +Operation <option>--add</option> + +Synopsis + + + nix-store + + paths + + + + +Description + +The operation adds the specified paths to +the Nix store. It prints the resulting paths in the Nix store on +standard output. + + + +Example + + +$ nix-store --add ./foo.c +/nix/store/m7lrha58ph6rcnv109yzx1nk1cj7k7zf-foo.c + + + + + + + +Operation <option>--add-fixed</option> + +Synopsis + + + nix-store + + + algorithm + paths + + + + +Description + +The operation adds the specified paths to +the Nix store. Unlike paths are registered using the +specified hashing algorithm, resulting in the same output path as a fixed output +derivation. This can be used for sources that are not available from a public +url or broke since the download expression was written. + + +This operation has the following options: + + + + + + + Use recursive instead of flat hashing mode, used when adding directories + to the store. + + + + + + + + + + +Example + + +$ nix-store --add-fixed sha256 ./hello-2.10.tar.gz +/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz + + + + + + + + + +Operation <option>--verify</option> + + + Synopsis + + nix-store + + + + + + +Description + +The operation verifies the internal +consistency of the Nix database, and the consistency between the Nix +database and the Nix store. Any inconsistencies encountered are +automatically repaired. Inconsistencies are generally the result of +the Nix store or database being modified by non-Nix tools, or of bugs +in Nix itself. + +This operation has the following options: + + + + + + Checks that the contents of every valid store path + has not been altered by computing a SHA-256 hash of the contents + and comparing it with the hash stored in the Nix database at build + time. Paths that have been modified are printed out. For large + stores, is obviously quite + slow. + + + + + + If any valid path is missing from the store, or + (if is given) the contents of a + valid path has been modified, then try to repair the path by + redownloading it. See nix-store --repair-path + for details. + + + + + + + + + + + + + + + +Operation <option>--verify-path</option> + + + Synopsis + + nix-store + + paths + + + +Description + +The operation compares the +contents of the given store paths to their cryptographic hashes stored +in Nix’s database. For every changed path, it prints a warning +message. The exit status is 0 if no path has changed, and 1 +otherwise. + + + +Example + +To verify the integrity of the svn command and all its dependencies: + + +$ nix-store --verify-path $(nix-store -qR $(which svn)) + + + + + + + + + + + +Operation <option>--repair-path</option> + + + Synopsis + + nix-store + + paths + + + +Description + +The operation attempts to +“repair” the specified paths by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +During repair, there is a very small time window during +which the old path (if it exists) is moved out of the way and replaced +with the new path. If repair is interrupted in between, then the +system may be left in a broken state (e.g., if the path contains a +critical system component like the GNU C Library). + + + +Example + + +$ nix-store --verify-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +path `/nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13' was modified! + expected hash `2db57715ae90b7e31ff1f2ecb8c12ec1cc43da920efcbe3b22763f36a1861588', + got `481c5aa5483ebc97c20457bb8bca24deea56550d3985cda0027f67fe54b808e4' + +$ nix-store --repair-path /nix/store/dj7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13 +fetching path `/nix/store/d7a81wsm1ijwwpkks3725661h3263p5-glibc-2.13'... +… + + + + + + + + + +Operation <option>--dump</option> + + + Synopsis + + nix-store + + path + + + +Description + +The operation produces a NAR (Nix +ARchive) file containing the contents of the file system tree rooted +at path. The archive is written to +standard output. + +A NAR archive is like a TAR or Zip archive, but it contains only +the information that Nix considers important. For instance, +timestamps are elided because all files in the Nix store have their +timestamp set to 0 anyway. Likewise, all permissions are left out +except for the execute bit, because all files in the Nix store have +644 or 755 permission. + +Also, a NAR archive is canonical, meaning +that “equal” paths always produce the same NAR archive. For instance, +directory entries are always sorted so that the actual on-disk order +doesn’t influence the result. This means that the cryptographic hash +of a NAR dump of a path is usable as a fingerprint of the contents of +the path. Indeed, the hashes of store paths stored in Nix’s database +(see nix-store -q +--hash) are SHA-256 hashes of the NAR dump of each +store path. + +NAR archives support filenames of unlimited length and 64-bit +file sizes. They can contain regular files, directories, and symbolic +links, but not other types of files (such as device nodes). + +A Nix archive can be unpacked using nix-store +--restore. + + + + + + + + + +Operation <option>--restore</option> + + + Synopsis + + nix-store + + path + + + +Description + +The operation unpacks a NAR archive +to path, which must not already exist. The +archive is read from standard input. + + + + + + + + + +Operation <option>--export</option> + + + Synopsis + + nix-store + + paths + + + +Description + +The operation writes a serialisation +of the specified store paths to standard output in a format that can +be imported into another Nix store with nix-store --import. This +is like nix-store +--dump, except that the NAR archive produced by that command +doesn’t contain the necessary meta-information to allow it to be +imported into another Nix store (namely, the set of references of the +path). + +This command does not produce a closure of +the specified paths, so if a store path references other store paths +that are missing in the target Nix store, the import will fail. To +copy a whole closure, do something like: + + +$ nix-store --export $(nix-store -qR paths) > out + +To import the whole closure again, run: + + +$ nix-store --import < out + + + + + + + + + + + +Operation <option>--import</option> + + + Synopsis + + nix-store + + + + +Description + +The operation reads a serialisation of +a set of store paths produced by nix-store --export from +standard input and adds those store paths to the Nix store. Paths +that already exist in the Nix store are ignored. If a path refers to +another path that doesn’t exist in the Nix store, the import +fails. + + + + + + + + + +Operation <option>--optimise</option> + + + Synopsis + + nix-store + + + + +Description + +The operation reduces Nix store disk +space usage by finding identical files in the store and hard-linking +them to each other. It typically reduces the size of the store by +something like 25-35%. Only regular files and symlinks are +hard-linked in this manner. Files are considered identical when they +have the same NAR archive serialisation: that is, regular files must +have the same contents and permission (executable or non-executable), +and symlinks must have the same contents. + +After completion, or when the command is interrupted, a report +on the achieved savings is printed on standard error. + +Use or to get some +progress indication. + + + +Example + + +$ nix-store --optimise +hashing files in `/nix/store/qhqx7l2f1kmwihc9bnxs7rc159hsxnf3-gcc-4.1.1' +... +541838819 bytes (516.74 MiB) freed by hard-linking 54143 files; +there are 114486 files with equal contents out of 215894 files in total + + + + + + + + + + +Operation <option>--read-log</option> + + + Synopsis + + nix-store + + + + + paths + + + +Description + +The operation prints the build log +of the specified store paths on standard output. The build log is +whatever the builder of a derivation wrote to standard output and +standard error. If a store path is not a derivation, the deriver of +the store path is used. + +Build logs are kept in +/nix/var/log/nix/drvs. However, there is no +guarantee that a build log is available for any particular store path. +For instance, if the path was downloaded as a pre-built binary through +a substitute, then the log is unavailable. + + + +Example + + +$ nix-store -l $(which ktorrent) +building /nix/store/dhc73pvzpnzxhdgpimsd9sw39di66ph1-ktorrent-2.2.1 +unpacking sources +unpacking source archive /nix/store/p8n1jpqs27mgkjw07pb5269717nzf5f8-ktorrent-2.2.1.tar.gz +ktorrent-2.2.1/ +ktorrent-2.2.1/NEWS +... + + + + + + + + + + +Operation <option>--dump-db</option> + + + Synopsis + + nix-store + + paths + + + +Description + +The operation writes a dump of the +Nix database to standard output. It can be loaded into an empty Nix +store using . This is useful for making +backups and when migrating to different database schemas. + +By default, will dump the entire Nix +database. When one or more store paths is passed, only the subset of +the Nix database for those store paths is dumped. As with +, the user is responsible for passing all the +store paths for a closure. See for an +example. + + + + + + + + +Operation <option>--load-db</option> + + + Synopsis + + nix-store + + + + +Description + +The operation reads a dump of the Nix +database created by from standard input and +loads it into the Nix database. + + + + + + + + +Operation <option>--print-env</option> + + + Synopsis + + nix-store + + drvpath + + + +Description + +The operation prints out the +environment of a derivation in a format that can be evaluated by a +shell. The command line arguments of the builder are placed in the +variable _args. + + + +Example + + +$ nix-store --print-env $(nix-instantiate '<nixpkgs>' -A firefox) +… +export src; src='/nix/store/plpj7qrwcz94z2psh6fchsi7s8yihc7k-firefox-12.0.source.tar.bz2' +export stdenv; stdenv='/nix/store/7c8asx3yfrg5dg1gzhzyq2236zfgibnn-stdenv' +export system; system='x86_64-linux' +export _args; _args='-e /nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25c-default-builder.sh' + + + + + + + + + +Operation <option>--generate-binary-cache-key</option> + + + Synopsis + + nix-store + + + + + + + + + +Description + +This command generates an Ed25519 key pair that can +be used to create a signed binary cache. It takes three mandatory +parameters: + + + + A key name, such as + cache.example.org-1, that is used to look up keys + on the client when it verifies signatures. It can be anything, but + it’s suggested to use the host name of your cache + (e.g. cache.example.org) with a suffix denoting + the number of the key (to be incremented every time you need to + revoke a key). + + The file name where the secret key is to be + stored. + + The file name where the public key is to be + stored. + + + + + + + + + + + + +Environment variables + + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/opt-common-syn.xml b/third_party/nix/doc/manual/command-ref/opt-common-syn.xml new file mode 100644 index 000000000..b610b54b9 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/opt-common-syn.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + + + + + + + + + number + + + + number + + + + number + + + + number + + + + + + + + + + + + + + + + + + path + + + + name + value + + + + diff --git a/third_party/nix/doc/manual/command-ref/opt-common.xml b/third_party/nix/doc/manual/command-ref/opt-common.xml new file mode 100644 index 000000000..b8a2f260e --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/opt-common.xml @@ -0,0 +1,366 @@ + + +Common Options + + +Most Nix commands accept the following command-line options: + + + + + + Prints out a summary of the command syntax and + exits. + + + + + + + Prints out the Nix version number on standard output + and exits. + + + + / + + + + Increases the level of verbosity of diagnostic messages + printed on standard error. For each Nix operation, the information + printed on standard output is well-defined; any diagnostic + information is printed on standard error, never on standard + output. + + This option may be specified repeatedly. Currently, the + following verbosity levels exist: + + + + 0 + “Errors only”: only print messages + explaining why the Nix invocation failed. + + + 1 + “Informational”: print + useful messages about what Nix is doing. + This is the default. + + + 2 + “Talkative”: print more informational + messages. + + + 3 + “Chatty”: print even more + informational messages. + + + 4 + “Debug”: print debug + information. + + + 5 + “Vomit”: print vast amounts of debug + information. + + + + + + + + + + + + + + Decreases the level of verbosity of diagnostic messages + printed on standard error. This is the inverse option to + / . + + + This option may be specified repeatedly. See the previous + verbosity levels list. + + + + + + + / + + By default, output written by builders to standard + output and standard error is echoed to the Nix command's standard + error. This option suppresses this behaviour. Note that the + builder's standard output and error are always written to a log file + in + prefix/nix/var/log/nix. + + + + + / +number + + + + Sets the maximum number of build jobs that Nix will + perform in parallel to the specified number. Specify + auto to use the number of CPUs in the system. + The default is specified by the max-jobs + configuration setting, which itself defaults to + 1. A higher value is useful on SMP systems or to + exploit I/O latency. + + Setting it to 0 disallows building on the local + machine, which is useful when you want builds to happen only on remote + builders. + + + + + + + + + Sets the value of the NIX_BUILD_CORES + environment variable in the invocation of builders. Builders can + use this variable at their discretion to control the maximum amount + of parallelism. For instance, in Nixpkgs, if the derivation + attribute enableParallelBuilding is set to + true, the builder passes the + flag to GNU Make. + It defaults to the value of the cores + configuration setting, if set, or 1 otherwise. + The value 0 means that the builder should use all + available CPU cores in the system. + + + + + + + Sets the maximum number of seconds that a builder + can go without producing any data on standard output or standard + error. The default is specified by the max-silent-time + configuration setting. 0 means no + time-out. + + + + + + Sets the maximum number of seconds that a builder + can run. The default is specified by the timeout + configuration setting. 0 means no + timeout. + + + + / + + Keep going in case of failed builds, to the + greatest extent possible. That is, if building an input of some + derivation fails, Nix will still build the other inputs, but not the + derivation itself. Without this option, Nix stops if any build + fails (except for builds of substitutes), possibly killing builds in + progress (in case of parallel or distributed builds). + + + + + / + + Specifies that in case of a build failure, the + temporary directory (usually in /tmp) in which + the build takes place should not be deleted. The path of the build + directory is printed as an informational message. + + + + + + + + + + Whenever Nix attempts to build a derivation for which + substitutes are known for each output path, but realising the output + paths through the substitutes fails, fall back on building the + derivation. + + The most common scenario in which this is useful is when we + have registered substitutes in order to perform binary distribution + from, say, a network repository. If the repository is down, the + realisation of the derivation will fail. When this option is + specified, Nix will build the derivation instead. Thus, + installation from binaries falls back on installation from source. + This option is not the default since it is generally not desirable + for a transient failure in obtaining the substitutes to lead to a + full build from source (with the related consumption of + resources). + + + + + + + + + + Disables the build hook mechanism. This allows to ignore remote + builders if they are setup on the machine. + + It's useful in cases where the bandwidth between the client and the + remote builder is too low. In that case it can take more time to upload the + sources to the remote builder and fetch back the result than to do the + computation locally. + + + + + + + + + + When this option is used, no attempt is made to open + the Nix database. Most Nix operations do need database access, so + those operations will fail. + + + + + name value + + This option is accepted by + nix-env, nix-instantiate and + nix-build. When evaluating Nix expressions, the + expression evaluator will automatically try to call functions that + it encounters. It can automatically call functions for which every + argument has a default value + (e.g., { argName ? + defaultValue }: + ...). With + , you can also call functions that have + arguments without a default value (or override a default value). + That is, if the evaluator encounters a function with an argument + named name, it will call it with value + value. + + For instance, the top-level default.nix in + Nixpkgs is actually a function: + + +{ # The system (e.g., `i686-linux') for which to build the packages. + system ? builtins.currentSystem + ... +}: ... + + So if you call this Nix expression (e.g., when you do + nix-env -i pkgname), + the function will be called automatically using the value builtins.currentSystem + for the system argument. You can override this + using , e.g., nix-env -i + pkgname --arg system + \"i686-freebsd\". (Note that since the argument is a Nix + string literal, you have to escape the quotes.) + + + + + name value + + This option is like , only the + value is not a Nix expression but a string. So instead of + --arg system \"i686-linux\" (the outer quotes are + to keep the shell happy) you can say --argstr system + i686-linux. + + + + + / +attrPath + + Select an attribute from the top-level Nix + expression being evaluated. (nix-env, + nix-instantiate, nix-build and + nix-shell only.) The attribute + path attrPath is a sequence of + attribute names separated by dots. For instance, given a top-level + Nix expression e, the attribute path + xorg.xorgserver would cause the expression + e.xorg.xorgserver to + be used. See nix-env + --install for some concrete examples. + + In addition to attribute names, you can also specify array + indices. For instance, the attribute path + foo.3.bar selects the bar + attribute of the fourth element of the array in the + foo attribute of the top-level + expression. + + + + + / + + Interpret the command line arguments as a list of + Nix expressions to be parsed and evaluated, rather than as a list + of file names of Nix expressions. + (nix-instantiate, nix-build + and nix-shell only.) + + + + + path + + Add a path to the Nix expression search path. This + option may be given multiple times. See the NIX_PATH environment variable for + information on the semantics of the Nix search path. Paths added + through take precedence over + NIX_PATH. + + + + + name value + + Set the Nix configuration option + name to value. + This overrides settings in the Nix configuration file (see + nix.conf5). + + + + + + + Fix corrupted or missing store paths by + redownloading or rebuilding them. Note that this is slow because it + requires computing a cryptographic hash of the contents of every + path in the closure of the build. Also note the warning under + nix-store --repair-path. + + + + + + + + diff --git a/third_party/nix/doc/manual/command-ref/opt-inst-syn.xml b/third_party/nix/doc/manual/command-ref/opt-inst-syn.xml new file mode 100644 index 000000000..e8c3f1ec6 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/opt-inst-syn.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + path + + diff --git a/third_party/nix/doc/manual/command-ref/utilities.xml b/third_party/nix/doc/manual/command-ref/utilities.xml new file mode 100644 index 000000000..893f5b5b5 --- /dev/null +++ b/third_party/nix/doc/manual/command-ref/utilities.xml @@ -0,0 +1,20 @@ + + +Utilities + +This section lists utilities that you can use when you +work with Nix. + + + + + + + + + + diff --git a/third_party/nix/doc/manual/expressions/advanced-attributes.xml b/third_party/nix/doc/manual/expressions/advanced-attributes.xml new file mode 100644 index 000000000..07b0d97d3 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/advanced-attributes.xml @@ -0,0 +1,340 @@ +
+ +Advanced Attributes + +Derivations can declare some infrequently used optional +attributes. + + + + allowedReferences + + The optional attribute + allowedReferences specifies a list of legal + references (dependencies) of the output of the builder. For + example, + + +allowedReferences = []; + + + enforces that the output of a derivation cannot have any runtime + dependencies on its inputs. To allow an output to have a runtime + dependency on itself, use "out" as a list item. + This is used in NixOS to check that generated files such as + initial ramdisks for booting Linux don’t have accidental + dependencies on other paths in the Nix store. + + + + + allowedRequisites + + This attribute is similar to + allowedReferences, but it specifies the legal + requisites of the whole closure, so all the dependencies + recursively. For example, + + +allowedRequisites = [ foobar ]; + + + enforces that the output of a derivation cannot have any other + runtime dependency than foobar, and in addition + it enforces that foobar itself doesn't + introduce any other dependency itself. + + + + disallowedReferences + + The optional attribute + disallowedReferences specifies a list of illegal + references (dependencies) of the output of the builder. For + example, + + +disallowedReferences = [ foo ]; + + + enforces that the output of a derivation cannot have a direct runtime + dependencies on the derivation foo. + + + + + disallowedRequisites + + This attribute is similar to + disallowedReferences, but it specifies illegal + requisites for the whole closure, so all the dependencies + recursively. For example, + + +disallowedRequisites = [ foobar ]; + + + enforces that the output of a derivation cannot have any + runtime dependency on foobar or any other derivation + depending recursively on foobar. + + + + + exportReferencesGraph + + This attribute allows builders access to the + references graph of their inputs. The attribute is a list of + inputs in the Nix store whose references graph the builder needs + to know. The value of this attribute should be a list of pairs + [ name1 + path1 name2 + path2 ... + ]. The references graph of each + pathN will be stored in a text file + nameN in the temporary build directory. + The text files have the format used by nix-store + --register-validity (with the deriver fields left + empty). For example, when the following derivation is built: + + +derivation { + ... + exportReferencesGraph = [ "libfoo-graph" libfoo ]; +}; + + + the references graph of libfoo is placed in the + file libfoo-graph in the temporary build + directory. + + exportReferencesGraph is useful for + builders that want to do something with the closure of a store + path. Examples include the builders in NixOS that generate the + initial ramdisk for booting Linux (a cpio + archive containing the closure of the boot script) and the + ISO-9660 image for the installation CD (which is populated with a + Nix store containing the closure of a bootable NixOS + configuration). + + + + + impureEnvVars + + This attribute allows you to specify a list of + environment variables that should be passed from the environment + of the calling user to the builder. Usually, the environment is + cleared completely when the builder is executed, but with this + attribute you can allow specific environment variables to be + passed unmodified. For example, fetchurl in + Nixpkgs has the line + + +impureEnvVars = [ "http_proxy" "https_proxy" ... ]; + + + to make it use the proxy server configuration specified by the + user in the environment variables http_proxy and + friends. + + This attribute is only allowed in fixed-output derivations, where + impurities such as these are okay since (the hash of) the output + is known in advance. It is ignored for all other + derivations. + + impureEnvVars implementation takes + environment variables from the current builder process. When a daemon is + building its environmental variables are used. Without the daemon, the + environmental variables come from the environment of the + nix-build. + + + + + + outputHash + outputHashAlgo + outputHashMode + + These attributes declare that the derivation is a + so-called fixed-output derivation, which + means that a cryptographic hash of the output is already known in + advance. When the build of a fixed-output derivation finishes, + Nix computes the cryptographic hash of the output and compares it + to the hash declared with these attributes. If there is a + mismatch, the build fails. + + The rationale for fixed-output derivations is derivations + such as those produced by the fetchurl + function. This function downloads a file from a given URL. To + ensure that the downloaded file has not been modified, the caller + must also specify a cryptographic hash of the file. For example, + + +fetchurl { + url = http://ftp.gnu.org/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; +} + + + It sometimes happens that the URL of the file changes, e.g., + because servers are reorganised or no longer available. We then + must update the call to fetchurl, e.g., + + +fetchurl { + url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; +} + + + If a fetchurl derivation was treated like a + normal derivation, the output paths of the derivation and + all derivations depending on it would change. + For instance, if we were to change the URL of the Glibc source + distribution in Nixpkgs (a package on which almost all other + packages depend) massive rebuilds would be needed. This is + unfortunate for a change which we know cannot have a real effect + as it propagates upwards through the dependency graph. + + For fixed-output derivations, on the other hand, the name of + the output path only depends on the outputHash* + and name attributes, while all other attributes + are ignored for the purpose of computing the output path. (The + name attribute is included because it is part + of the path.) + + As an example, here is the (simplified) Nix expression for + fetchurl: + + +{ stdenv, curl }: # The curl program is used for downloading. + +{ url, sha256 }: + +stdenv.mkDerivation { + name = baseNameOf (toString url); + builder = ./builder.sh; + buildInputs = [ curl ]; + + # This is a fixed-output derivation; the output must be a regular + # file with SHA256 hash sha256. + outputHashMode = "flat"; + outputHashAlgo = "sha256"; + outputHash = sha256; + + inherit url; +} + + + + + The outputHashAlgo attribute specifies + the hash algorithm used to compute the hash. It can currently be + "sha1", "sha256" or + "sha512". + + The outputHashMode attribute determines + how the hash is computed. It must be one of the following two + values: + + + + "flat" + + The output must be a non-executable regular + file. If it isn’t, the build fails. The hash is simply + computed over the contents of that file (so it’s equal to what + Unix commands like sha256sum or + sha1sum produce). + + This is the default. + + + + "recursive" + + The hash is computed over the NAR archive dump + of the output (i.e., the result of nix-store + --dump). In this case, the output can be + anything, including a directory tree. + + + + + + + + The outputHash attribute, finally, must + be a string containing the hash in either hexadecimal or base-32 + notation. (See the nix-hash command + for information about converting to and from base-32 + notation.) + + + + + passAsFile + + A list of names of attributes that should be + passed via files rather than environment variables. For example, + if you have + + +passAsFile = ["big"]; +big = "a very long string"; + + + then when the builder runs, the environment variable + bigPath will contain the absolute path to a + temporary file containing a very long + string. That is, for any attribute + x listed in + passAsFile, Nix will pass an environment + variable xPath holding + the path of the file containing the value of attribute + x. This is useful when you need to pass + large strings to a builder, since most operating systems impose a + limit on the size of the environment (typically, a few hundred + kilobyte). + + + + + preferLocalBuild + + If this attribute is set to + true and distributed building is + enabled, then, if possible, the derivaton will be built + locally instead of forwarded to a remote machine. This is + appropriate for trivial builders where the cost of doing a + download or remote build would exceed the cost of building + locally. + + + + + allowSubstitutes + + If this attribute is set to + false, then Nix will always build this + derivation; it will not try to substitute its outputs. This is + useful for very trivial derivations (such as + writeText in Nixpkgs) that are cheaper to + build than to substitute from a binary cache. + + + + + + +
diff --git a/third_party/nix/doc/manual/expressions/arguments-variables.xml b/third_party/nix/doc/manual/expressions/arguments-variables.xml new file mode 100644 index 000000000..bf60cb7ee --- /dev/null +++ b/third_party/nix/doc/manual/expressions/arguments-variables.xml @@ -0,0 +1,121 @@ +
+ +Arguments and Variables + + + +Composing GNU Hello +(<filename>all-packages.nix</filename>) + +... + +rec { + + hello = import ../applications/misc/hello/ex-1 { + inherit fetchurl stdenv perl; + }; + + perl = import ../development/interpreters/perl { + inherit fetchurl stdenv; + }; + + fetchurl = import ../build-support/fetchurl { + inherit stdenv; ... + }; + + stdenv = ...; + +} + + + +The Nix expression in is a +function; it is missing some arguments that have to be filled in +somewhere. In the Nix Packages collection this is done in the file +pkgs/top-level/all-packages.nix, where all +Nix expressions for packages are imported and called with the +appropriate arguments. shows +some fragments of +all-packages.nix. + + + + + + This file defines a set of attributes, all of which are + concrete derivations (i.e., not functions). In fact, we define a + mutually recursive set of attributes. That + is, the attributes can refer to each other. This is precisely + what we want since we want to plug the + various packages into each other. + + + + + + Here we import the Nix expression for + GNU Hello. The import operation just loads and returns the + specified Nix expression. In fact, we could just have put the + contents of in + all-packages.nix at this point. That + would be completely equivalent, but it would make the file rather + bulky. + + Note that we refer to + ../applications/misc/hello/ex-1, not + ../applications/misc/hello/ex-1/default.nix. + When you try to import a directory, Nix automatically appends + /default.nix to the file name. + + + + + + This is where the actual composition takes place. Here we + call the function imported from + ../applications/misc/hello/ex-1 with a set + containing the things that the function expects, namely + fetchurl, stdenv, and + perl. We use inherit again to use the + attributes defined in the surrounding scope (we could also have + written fetchurl = fetchurl;, etc.). + + The result of this function call is an actual derivation + that can be built by Nix (since when we fill in the arguments of + the function, what we get is its body, which is the call to + stdenv.mkDerivation in ). + + Nixpkgs has a convenience function + callPackage that imports and calls a + function, filling in any missing arguments by passing the + corresponding attribute from the Nixpkgs set, like this: + + +hello = callPackage ../applications/misc/hello/ex-1 { }; + + + If necessary, you can set or override arguments: + + +hello = callPackage ../applications/misc/hello/ex-1 { stdenv = myStdenv; }; + + + + + + + + + Likewise, we have to instantiate Perl, + fetchurl, and the standard environment. + + + + + +
\ No newline at end of file diff --git a/third_party/nix/doc/manual/expressions/build-script.xml b/third_party/nix/doc/manual/expressions/build-script.xml new file mode 100644 index 000000000..7bad8f808 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/build-script.xml @@ -0,0 +1,119 @@ +
+ +Build Script + +Build script for GNU Hello +(<filename>builder.sh</filename>) + +source $stdenv/setup + +PATH=$perl/bin:$PATH + +tar xvfz $src +cd hello-* +./configure --prefix=$out +make +make install + + + shows the builder referenced +from Hello's Nix expression (stored in +pkgs/applications/misc/hello/ex-1/builder.sh). +The builder can actually be made a lot shorter by using the +generic builder functions provided by +stdenv, but here we write out the build steps to +elucidate what a builder does. It performs the following +steps: + + + + + + When Nix runs a builder, it initially completely clears the + environment (except for the attributes declared in the + derivation). For instance, the PATH variable is + emptyActually, it's initialised to + /path-not-set to prevent Bash from setting it + to a default value.. This is done to prevent + undeclared inputs from being used in the build process. If for + example the PATH contained + /usr/bin, then you might accidentally use + /usr/bin/gcc. + + So the first step is to set up the environment. This is + done by calling the setup script of the + standard environment. The environment variable + stdenv points to the location of the standard + environment being used. (It wasn't specified explicitly as an + attribute in , but + mkDerivation adds it automatically.) + + + + + + Since Hello needs Perl, we have to make sure that Perl is in + the PATH. The perl environment + variable points to the location of the Perl package (since it + was passed in as an attribute to the derivation), so + $perl/bin is the + directory containing the Perl interpreter. + + + + + + Now we have to unpack the sources. The + src attribute was bound to the result of + fetching the Hello source tarball from the network, so the + src environment variable points to the location in + the Nix store to which the tarball was downloaded. After + unpacking, we cd to the resulting source + directory. + + The whole build is performed in a temporary directory + created in /tmp, by the way. This directory is + removed after the builder finishes, so there is no need to clean + up the sources afterwards. Also, the temporary directory is + always newly created, so you don't have to worry about files from + previous builds interfering with the current build. + + + + + + GNU Hello is a typical Autoconf-based package, so we first + have to run its configure script. In Nix + every package is stored in a separate location in the Nix store, + for instance + /nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1. + Nix computes this path by cryptographically hashing all attributes + of the derivation. The path is passed to the builder through the + out environment variable. So here we give + configure the parameter + --prefix=$out to cause Hello to be installed in + the expected location. + + + + + + Finally we build Hello (make) and install + it into the location specified by out + (make install). + + + + + +If you are wondering about the absence of error checking on the +result of various commands called in the builder: this is because the +shell script is evaluated with Bash's option, +which causes the script to be aborted if any command fails without an +error check. + +
\ No newline at end of file diff --git a/third_party/nix/doc/manual/expressions/builder-syntax.xml b/third_party/nix/doc/manual/expressions/builder-syntax.xml new file mode 100644 index 000000000..e51bade44 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/builder-syntax.xml @@ -0,0 +1,119 @@ +
+ +Builder Syntax + +Build script for GNU Hello +(<filename>builder.sh</filename>) + +source $stdenv/setup + +PATH=$perl/bin:$PATH + +tar xvfz $src +cd hello-* +./configure --prefix=$out +make +make install + + + shows the builder referenced +from Hello's Nix expression (stored in +pkgs/applications/misc/hello/ex-1/builder.sh). +The builder can actually be made a lot shorter by using the +generic builder functions provided by +stdenv, but here we write out the build steps to +elucidate what a builder does. It performs the following +steps: + + + + + + When Nix runs a builder, it initially completely clears the + environment (except for the attributes declared in the + derivation). For instance, the PATH variable is + emptyActually, it's initialised to + /path-not-set to prevent Bash from setting it + to a default value.. This is done to prevent + undeclared inputs from being used in the build process. If for + example the PATH contained + /usr/bin, then you might accidentally use + /usr/bin/gcc. + + So the first step is to set up the environment. This is + done by calling the setup script of the + standard environment. The environment variable + stdenv points to the location of the standard + environment being used. (It wasn't specified explicitly as an + attribute in , but + mkDerivation adds it automatically.) + + + + + + Since Hello needs Perl, we have to make sure that Perl is in + the PATH. The perl environment + variable points to the location of the Perl package (since it + was passed in as an attribute to the derivation), so + $perl/bin is the + directory containing the Perl interpreter. + + + + + + Now we have to unpack the sources. The + src attribute was bound to the result of + fetching the Hello source tarball from the network, so the + src environment variable points to the location in + the Nix store to which the tarball was downloaded. After + unpacking, we cd to the resulting source + directory. + + The whole build is performed in a temporary directory + created in /tmp, by the way. This directory is + removed after the builder finishes, so there is no need to clean + up the sources afterwards. Also, the temporary directory is + always newly created, so you don't have to worry about files from + previous builds interfering with the current build. + + + + + + GNU Hello is a typical Autoconf-based package, so we first + have to run its configure script. In Nix + every package is stored in a separate location in the Nix store, + for instance + /nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1. + Nix computes this path by cryptographically hashing all attributes + of the derivation. The path is passed to the builder through the + out environment variable. So here we give + configure the parameter + --prefix=$out to cause Hello to be installed in + the expected location. + + + + + + Finally we build Hello (make) and install + it into the location specified by out + (make install). + + + + + +If you are wondering about the absence of error checking on the +result of various commands called in the builder: this is because the +shell script is evaluated with Bash's option, +which causes the script to be aborted if any command fails without an +error check. + +
\ No newline at end of file diff --git a/third_party/nix/doc/manual/expressions/builtins.xml b/third_party/nix/doc/manual/expressions/builtins.xml new file mode 100644 index 000000000..394e1fc32 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/builtins.xml @@ -0,0 +1,1658 @@ +
+ +Built-in Functions + +This section lists the functions and constants built into the +Nix expression evaluator. (The built-in function +derivation is discussed above.) Some built-ins, +such as derivation, are always in scope of every +Nix expression; you can just access them right away. But to prevent +polluting the namespace too much, most built-ins are not in scope. +Instead, you can access them through the builtins +built-in value, which is a set that contains all built-in functions +and values. For instance, derivation is also +available as builtins.derivation. + + + + + + + abort s + builtins.abort s + + Abort Nix expression evaluation, print error + message s. + + + + + + builtins.add + e1 e2 + + + Return the sum of the numbers + e1 and + e2. + + + + + + builtins.all + pred list + + Return true if the function + pred returns true + for all elements of list, + and false otherwise. + + + + + + builtins.any + pred list + + Return true if the function + pred returns true + for at least one element of list, + and false otherwise. + + + + + + builtins.attrNames + set + + Return the names of the attributes in the set + set in an alphabetically sorted list. For instance, + builtins.attrNames { y = 1; x = "foo"; } + evaluates to [ "x" "y" ]. + + + + + + builtins.attrValues + set + + Return the values of the attributes in the set + set in the order corresponding to the + sorted attribute names. + + + + + + baseNameOf s + + Return the base name of the + string s, that is, everything following + the final slash in the string. This is similar to the GNU + basename command. + + + + + + builtins.bitAnd + e1 e2 + + Return the bitwise AND of the integers + e1 and + e2. + + + + + + builtins.bitOr + e1 e2 + + Return the bitwise OR of the integers + e1 and + e2. + + + + + + builtins.bitXor + e1 e2 + + Return the bitwise XOR of the integers + e1 and + e2. + + + + + + builtins + + The set builtins contains all + the built-in functions and values. You can use + builtins to test for the availability of + features in the Nix installation, e.g., + + +if builtins ? getEnv then builtins.getEnv "PATH" else "" + + This allows a Nix expression to fall back gracefully on older Nix + installations that don’t have the desired built-in + function. + + + + + + builtins.compareVersions + s1 s2 + + Compare two strings representing versions and + return -1 if version + s1 is older than version + s2, 0 if they are + the same, and 1 if + s1 is newer than + s2. The version comparison algorithm + is the same as the one used by nix-env + -u. + + + + + + builtins.concatLists + lists + + Concatenate a list of lists into a single + list. + + + + + builtins.concatStringsSep + separator list + + Concatenate a list of strings with a separator + between each element, e.g. concatStringsSep "/" + ["usr" "local" "bin"] == "usr/local/bin" + + + + + builtins.currentSystem + + The built-in value currentSystem + evaluates to the Nix platform identifier for the Nix installation + on which the expression is being evaluated, such as + "i686-linux" or + "x86_64-darwin". + + + + + + + + + + + + builtins.deepSeq + e1 e2 + + This is like seq + e1 + e2, except that + e1 is evaluated + deeply: if it’s a list or set, its elements + or attributes are also evaluated recursively. + + + + + + derivation + attrs + builtins.derivation + attrs + + derivation is described in + . + + + + + + dirOf s + builtins.dirOf s + + Return the directory part of the string + s, that is, everything before the final + slash in the string. This is similar to the GNU + dirname command. + + + + + + builtins.div + e1 e2 + + Return the quotient of the numbers + e1 and + e2. + + + + + builtins.elem + x xs + + Return true if a value equal to + x occurs in the list + xs, and false + otherwise. + + + + + + builtins.elemAt + xs n + + Return element n from + the list xs. Elements are counted + starting from 0. A fatal error occurs if the index is out of + bounds. + + + + + + builtins.fetchurl + url + + Download the specified URL and return the path of + the downloaded file. This function is not available if restricted evaluation mode is + enabled. + + + + + + fetchTarball + url + builtins.fetchTarball + url + + Download the specified URL, unpack it and return + the path of the unpacked tree. The file must be a tape archive + (.tar) compressed with + gzip, bzip2 or + xz. The top-level path component of the files + in the tarball is removed, so it is best if the tarball contains a + single directory at top level. The typical use of the function is + to obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. + + +with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; + +stdenv.mkDerivation { … } + + + + The fetched tarball is cached for a certain amount of time + (1 hour by default) in ~/.cache/nix/tarballs/. + You can change the cache timeout either on the command line with + or + in the Nix configuration file with this option: + number of seconds to cache. + + + Note that when obtaining the hash with nix-prefetch-url + the option --unpack is required. + + + This function can also verify the contents against a hash. + In that case, the function takes a set instead of a URL. The set + requires the attribute url and the attribute + sha256, e.g. + + +with import (fetchTarball { + url = https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz; + sha256 = "1jppksrfvbk5ypiqdz4cddxdl8z6zyzdb2srq8fcffr327ld5jj2"; +}) {}; + +stdenv.mkDerivation { … } + + + + + This function is not available if restricted evaluation mode is + enabled. + + + + + + builtins.fetchGit + args + + + + + Fetch a path from git. args can be + a URL, in which case the HEAD of the repo at that URL is + fetched. Otherwise, it can be an attribute with the following + attributes (all except url optional): + + + + + url + + + The URL of the repo. + + + + + name + + + The name of the directory the repo should be exported to + in the store. Defaults to the basename of the URL. + + + + + rev + + + The git revision to fetch. Defaults to the tip of + ref. + + + + + ref + + + The git ref to look for the requested revision under. + This is often a branch or tag name. Defaults to + HEAD. + + + + By default, the ref value is prefixed + with refs/heads/. As of Nix 2.3.0 + Nix will not prefix refs/heads/ if + ref starts with refs/. + + + + + + + Fetching a private repository over SSH + builtins.fetchGit { + url = "git@github.com:my-secret/repository.git"; + ref = "master"; + rev = "adab8b916a45068c044658c4158d81878f9ed1c3"; +} + + + + Fetching an arbitrary ref + builtins.fetchGit { + url = "https://github.com/NixOS/nix.git"; + ref = "refs/heads/0.5-release"; +} + + + + Fetching a repository's specific commit on an arbitrary branch + + If the revision you're looking for is in the default branch + of the git repository you don't strictly need to specify + the branch name in the ref attribute. + + + However, if the revision you're looking for is in a future + branch for the non-default branch you will need to specify + the the ref attribute as well. + + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; + ref = "1.11-maintenance"; +} + + + It is nice to always specify the branch which a revision + belongs to. Without the branch being specified, the + fetcher might fail if the default branch changes. + Additionally, it can be confusing to try a commit from a + non-default branch and see the fetch fail. If the branch + is specified the fault is much more obvious. + + + + + + Fetching a repository's specific commit on the default branch + + If the revision you're looking for is in the default branch + of the git repository you may omit the + ref attribute. + + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + rev = "841fcbd04755c7a2865c51c1e2d3b045976b7452"; +} + + + + Fetching a tag + builtins.fetchGit { + url = "https://github.com/nixos/nix.git"; + ref = "refs/tags/1.9"; +} + + + + Fetching the latest version of a remote branch + + builtins.fetchGit can behave impurely + fetch the latest version of a remote branch. + + Nix will refetch the branch in accordance to + . + This behavior is disabled in + Pure evaluation mode. + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + ref = "master"; +} + + + + + + builtins.filter + f xs + + Return a list consisting of the elements of + xs for which the function + f returns + true. + + + + + + builtins.filterSource + e1 e2 + + + + This function allows you to copy sources into the Nix + store while filtering certain files. For instance, suppose that + you want to use the directory source-dir as + an input to a Nix expression, e.g. + + +stdenv.mkDerivation { + ... + src = ./source-dir; +} + + + However, if source-dir is a Subversion + working copy, then all those annoying .svn + subdirectories will also be copied to the store. Worse, the + contents of those directories may change a lot, causing lots of + spurious rebuilds. With filterSource you + can filter out the .svn directories: + + + src = builtins.filterSource + (path: type: type != "directory" || baseNameOf path != ".svn") + ./source-dir; + + + + + Thus, the first argument e1 + must be a predicate function that is called for each regular + file, directory or symlink in the source tree + e2. If the function returns + true, the file is copied to the Nix store, + otherwise it is omitted. The function is called with two + arguments. The first is the full path of the file. The second + is a string that identifies the type of the file, which is + either "regular", + "directory", "symlink" or + "unknown" (for other kinds of files such as + device nodes or fifos — but note that those cannot be copied to + the Nix store, so if the predicate returns + true for them, the copy will fail). If you + exclude a directory, the entire corresponding subtree of + e2 will be excluded. + + + + + + + + builtins.foldl’ + op nul list + + Reduce a list by applying a binary operator, from + left to right, e.g. foldl’ op nul [x0 x1 x2 ...] = op (op + (op nul x0) x1) x2) .... The operator is applied + strictly, i.e., its arguments are evaluated first. For example, + foldl’ (x: y: x + y) 0 [1 2 3] evaluates to + 6. + + + + + + builtins.functionArgs + f + + + Return a set containing the names of the formal arguments expected + by the function f. + The value of each attribute is a Boolean denoting whether the corresponding + argument has a default value. For instance, + functionArgs ({ x, y ? 123}: ...) = { x = false; y = true; }. + + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + functionArgs (x: ...) = { }. + + + + + + builtins.fromJSON e + + Convert a JSON string to a Nix + value. For example, + + +builtins.fromJSON ''{"x": [1, 2, 3], "y": null}'' + + + returns the value { x = [ 1 2 3 ]; y = null; + }. + + + + + + builtins.genList + generator length + + Generate list of size + length, with each element + i equal to the value returned by + generator i. For + example, + + +builtins.genList (x: x * x) 5 + + + returns the list [ 0 1 4 9 16 ]. + + + + + + builtins.getAttr + s set + + getAttr returns the attribute + named s from + set. Evaluation aborts if the + attribute doesn’t exist. This is a dynamic version of the + . operator, since s + is an expression rather than an identifier. + + + + + + builtins.getEnv + s + + getEnv returns the value of + the environment variable s, or an empty + string if the variable doesn’t exist. This function should be + used with care, as it can introduce all sorts of nasty environment + dependencies in your Nix expression. + + getEnv is used in Nix Packages to + locate the file ~/.nixpkgs/config.nix, which + contains user-local settings for Nix Packages. (That is, it does + a getEnv "HOME" to locate the user’s home + directory.) + + + + + + builtins.hasAttr + s set + + hasAttr returns + true if set has an + attribute named s, and + false otherwise. This is a dynamic version of + the ? operator, since + s is an expression rather than an + identifier. + + + + + + builtins.hashString + type s + + Return a base-16 representation of the + cryptographic hash of string s. The + hash algorithm specified by type must + be one of "md5", "sha1", + "sha256" or "sha512". + + + + + + builtins.hashFile + type p + + Return a base-16 representation of the + cryptographic hash of the file at path p. The + hash algorithm specified by type must + be one of "md5", "sha1", + "sha256" or "sha512". + + + + + + builtins.head + list + + Return the first element of a list; abort + evaluation if the argument isn’t a list or is an empty list. You + can test whether a list is empty by comparing it with + []. + + + + + + import + path + builtins.import + path + + Load, parse and return the Nix expression in the + file path. If path + is a directory, the file default.nix + in that directory is loaded. Evaluation aborts if the + file doesn’t exist or contains an incorrect Nix expression. + import implements Nix’s module system: you + can put any Nix expression (such as a set or a function) in a + separate file, and use it from Nix expressions in other + files. + + Unlike some languages, import is a regular + function in Nix. Paths using the angle bracket syntax (e.g., + import <foo>) are normal path + values (see ). + + A Nix expression loaded by import must + not contain any free variables (identifiers + that are not defined in the Nix expression itself and are not + built-in). Therefore, it cannot refer to variables that are in + scope at the call site. For instance, if you have a calling + expression + + +rec { + x = 123; + y = import ./foo.nix; +} + + then the following foo.nix will give an + error: + + +x + 456 + + since x is not in scope in + foo.nix. If you want x + to be available in foo.nix, you should pass + it as a function argument: + + +rec { + x = 123; + y = import ./foo.nix x; +} + + and + + +x: x + 456 + + (The function argument doesn’t have to be called + x in foo.nix; any name + would work.) + + + + + + builtins.intersectAttrs + e1 e2 + + Return a set consisting of the attributes in the + set e2 that also exist in the set + e1. + + + + + + builtins.isAttrs + e + + Return true if + e evaluates to a set, and + false otherwise. + + + + + + builtins.isList + e + + Return true if + e evaluates to a list, and + false otherwise. + + + + + builtins.isFunction + e + + Return true if + e evaluates to a function, and + false otherwise. + + + + + + builtins.isString + e + + Return true if + e evaluates to a string, and + false otherwise. + + + + + + builtins.isInt + e + + Return true if + e evaluates to an int, and + false otherwise. + + + + + + builtins.isFloat + e + + Return true if + e evaluates to a float, and + false otherwise. + + + + + + builtins.isBool + e + + Return true if + e evaluates to a bool, and + false otherwise. + + + + builtins.isPath + e + + Return true if + e evaluates to a path, and + false otherwise. + + + + + isNull + e + builtins.isNull + e + + Return true if + e evaluates to null, + and false otherwise. + + This function is deprecated; + just write e == null instead. + + + + + + + + builtins.length + e + + Return the length of the list + e. + + + + + + builtins.lessThan + e1 e2 + + Return true if the number + e1 is less than the number + e2, and false + otherwise. Evaluation aborts if either + e1 or e2 + does not evaluate to a number. + + + + + + builtins.listToAttrs + e + + Construct a set from a list specifying the names + and values of each attribute. Each element of the list should be + a set consisting of a string-valued attribute + name specifying the name of the attribute, and + an attribute value specifying its value. + Example: + + +builtins.listToAttrs + [ { name = "foo"; value = 123; } + { name = "bar"; value = 456; } + ] + + + evaluates to + + +{ foo = 123; bar = 456; } + + + + + + + + map + f list + builtins.map + f list + + Apply the function f to + each element in the list list. For + example, + + +map (x: "foo" + x) [ "bar" "bla" "abc" ] + + evaluates to [ "foobar" "foobla" "fooabc" + ]. + + + + + + builtins.match + regex str + + Returns a list if the extended + POSIX regular expression regex + matches str precisely, otherwise returns + null. Each item in the list is a regex group. + + +builtins.match "ab" "abc" + + +Evaluates to null. + + +builtins.match "abc" "abc" + + +Evaluates to [ ]. + + +builtins.match "a(b)(c)" "abc" + + +Evaluates to [ "b" "c" ]. + + +builtins.match "[[:space:]]+([[:upper:]]+)[[:space:]]+" " FOO " + + +Evaluates to [ "foo" ]. + + + + + + builtins.mul + e1 e2 + + Return the product of the numbers + e1 and + e2. + + + + + + builtins.parseDrvName + s + + Split the string s into + a package name and version. The package name is everything up to + but not including the first dash followed by a digit, and the + version is everything following that dash. The result is returned + in a set { name, version }. Thus, + builtins.parseDrvName "nix-0.12pre12876" + returns { name = "nix"; version = "0.12pre12876"; + }. + + + + + + builtins.path + args + + + + + An enrichment of the built-in path type, based on the attributes + present in args. All are optional + except path: + + + + + path + + The underlying path. + + + + name + + + The name of the path when added to the store. This can + used to reference paths that have nix-illegal characters + in their names, like @. + + + + + filter + + + A function of the type expected by + builtins.filterSource, + with the same semantics. + + + + + recursive + + + When false, when + path is added to the store it is with a + flat hash, rather than a hash of the NAR serialization of + the file. Thus, path must refer to a + regular file, not a directory. This allows similar + behavior to fetchurl. Defaults to + true. + + + + + sha256 + + + When provided, this is the expected hash of the file at + the path. Evaluation will fail if the hash is incorrect, + and providing a hash allows + builtins.path to be used even when the + pure-eval nix config option is on. + + + + + + + + + builtins.pathExists + path + + Return true if the path + path exists at evaluation time, and + false otherwise. + + + + + builtins.placeholder + output + + Return a placeholder string for the specified + output that will be substituted by the + corresponding output path at build time. Typical outputs would be + "out", "bin" or + "dev". + + + + builtins.readDir + path + + Return the contents of the directory + path as a set mapping directory entries + to the corresponding file type. For instance, if directory + A contains a regular file + B and another directory + C, then builtins.readDir + ./A will return the set + + +{ B = "regular"; C = "directory"; } + + The possible values for the file type are + "regular", "directory", + "symlink" and + "unknown". + + + + + + builtins.readFile + path + + Return the contents of the file + path as a string. + + + + + + removeAttrs + set list + builtins.removeAttrs + set list + + Remove the attributes listed in + list from + set. The attributes don’t have to + exist in set. For instance, + + +removeAttrs { x = 1; y = 2; z = 3; } [ "a" "x" "z" ] + + evaluates to { y = 2; }. + + + + + + builtins.replaceStrings + from to s + + Given string s, replace + every occurrence of the strings in from + with the corresponding string in + to. For example, + + +builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" + + + evaluates to "fabir". + + + + + + builtins.seq + e1 e2 + + Evaluate e1, then + evaluate and return e2. This ensures + that a computation is strict in the value of + e1. + + + + + + builtins.sort + comparator list + + Return list in sorted + order. It repeatedly calls the function + comparator with two elements. The + comparator should return true if the first + element is less than the second, and false + otherwise. For example, + + +builtins.sort builtins.lessThan [ 483 249 526 147 42 77 ] + + + produces the list [ 42 77 147 249 483 526 + ]. + + This is a stable sort: it preserves the relative order of + elements deemed equal by the comparator. + + + + + + builtins.split + regex str + + Returns a list composed of non matched strings interleaved + with the lists of the extended + POSIX regular expression regex matches + of str. Each item in the lists of matched + sequences is a regex group. + + +builtins.split "(a)b" "abc" + + +Evaluates to [ "" [ "a" ] "c" ]. + + +builtins.split "([ac])" "abc" + + +Evaluates to [ "" [ "a" ] "b" [ "c" ] "" ]. + + +builtins.split "(a)|(c)" "abc" + + +Evaluates to [ "" [ "a" null ] "b" [ null "c" ] "" ]. + + +builtins.split "([[:upper:]]+)" " FOO " + + +Evaluates to [ " " [ "FOO" ] " " ]. + + + + + + + builtins.splitVersion + s + + Split a string representing a version into its + components, by the same version splitting logic underlying the + version comparison in + nix-env -u. + + + + + + builtins.stringLength + e + + Return the length of the string + e. If e is + not a string, evaluation is aborted. + + + + + + builtins.sub + e1 e2 + + Return the difference between the numbers + e1 and + e2. + + + + + + builtins.substring + start len + s + + Return the substring of + s from character position + start (zero-based) up to but not + including start + len. If + start is greater than the length of the + string, an empty string is returned, and if start + + len lies beyond the end of the string, only the + substring up to the end of the string is returned. + start must be + non-negative. For example, + + +builtins.substring 0 3 "nixos" + + + evaluates to "nix". + + + + + + + builtins.tail + list + + Return the second to last elements of a list; + abort evaluation if the argument isn’t a list or is an empty + list. + + + + + + throw + s + builtins.throw + s + + Throw an error message + s. This usually aborts Nix expression + evaluation, but in nix-env -qa and other + commands that try to evaluate a set of derivations to get + information about those derivations, a derivation that throws an + error is silently skipped (which is not the case for + abort). + + + + + + builtins.toFile + name + s + + Store the string s in a + file in the Nix store and return its path. The file has suffix + name. This file can be used as an + input to derivations. One application is to write builders + “inline”. For instance, the following Nix expression combines + and into one file: + + +{ stdenv, fetchurl, perl }: + +stdenv.mkDerivation { + name = "hello-2.1.1"; + + builder = builtins.toFile "builder.sh" " + source $stdenv/setup + + PATH=$perl/bin:$PATH + + tar xvfz $src + cd hello-* + ./configure --prefix=$out + make + make install + "; + + src = fetchurl { + url = http://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + }; + inherit perl; +} + + + + It is even possible for one file to refer to another, e.g., + + + builder = let + configFile = builtins.toFile "foo.conf" " + # This is some dummy configuration file. + ... + "; + in builtins.toFile "builder.sh" " + source $stdenv/setup + ... + cp ${configFile} $out/etc/foo.conf + "; + + Note that ${configFile} is an antiquotation + (see ), so the result of the + expression configFile (i.e., a path like + /nix/store/m7p7jfny445k...-foo.conf) will be + spliced into the resulting string. + + It is however not allowed to have files + mutually referring to each other, like so: + + +let + foo = builtins.toFile "foo" "...${bar}..."; + bar = builtins.toFile "bar" "...${foo}..."; +in foo + + This is not allowed because it would cause a cyclic dependency in + the computation of the cryptographic hashes for + foo and bar. + It is also not possible to reference the result of a derivation. + If you are using Nixpkgs, the writeTextFile function is able to + do that. + + + + + + builtins.toJSON e + + Return a string containing a JSON representation + of e. Strings, integers, floats, booleans, + nulls and lists are mapped to their JSON equivalents. Sets + (except derivations) are represented as objects. Derivations are + translated to a JSON string containing the derivation’s output + path. Paths are copied to the store and represented as a JSON + string of the resulting store path. + + + + + + builtins.toPath s + + DEPRECATED. Use /. + "/path" + to convert a string into an absolute path. For relative paths, + use ./. + "/path". + + + + + + + toString e + builtins.toString e + + Convert the expression + e to a string. + e can be: + + A string (in which case the string is returned unmodified). + A path (e.g., toString /foo/bar yields "/foo/bar". + A set containing { __toString = self: ...; }. + An integer. + A list, in which case the string representations of its elements are joined with spaces. + A Boolean (false yields "", true yields "1"). + null, which yields the empty string. + + + + + + + + builtins.toXML e + + Return a string containing an XML representation + of e. The main application for + toXML is to communicate information with the + builder in a more structured format than plain environment + variables. + + + + shows an example where this is + the case. The builder is supposed to generate the configuration + file for a Jetty + servlet container. A servlet container contains a number + of servlets (*.war files) each exported under + a specific URI prefix. So the servlet configuration is a list of + sets containing the path and + war of the servlet (). This kind of information is + difficult to communicate with the normal method of passing + information through an environment variable, which just + concatenates everything together into a string (which might just + work in this case, but wouldn’t work if fields are optional or + contain lists themselves). Instead the Nix expression is + converted to an XML representation with + toXML, which is unambiguous and can easily be + processed with the appropriate tools. For instance, in the + example an XSLT stylesheet () is applied to it () to + generate the XML configuration file for the Jetty server. The XML + representation produced from by toXML is shown in . + + Note that uses the toFile built-in to write the + builder and the stylesheet “inline” in the Nix expression. The + path of the stylesheet is spliced into the builder at + xsltproc ${stylesheet} + .... + + Passing information to a builder + using <function>toXML</function> + + $out/server-conf.xml]]> + + + + + + + + + + + + + "; + + servlets = builtins.toXML []]> + + + + XML representation produced by + <function>toXML</function> + + + + + + + + + + + + + + + + + + + + + +]]> + + + + + + + + + + builtins.trace + e1 e2 + + Evaluate e1 and print its + abstract syntax representation on standard error. Then return + e2. This function is useful for + debugging. + + + + + builtins.tryEval + e + + Try to shallowly evaluate e. + Return a set containing the attributes success + (true if e evaluated + successfully, false if an error was thrown) and + value, equalling e + if successful and false otherwise. Note that this + doesn't evaluate e deeply, so + let e = { x = throw ""; }; in (builtins.tryEval e).success + will be true. Using builtins.deepSeq + one can get the expected result: let e = { x = throw ""; + }; in (builtins.tryEval (builtins.deepSeq e e)).success will be + false. + + + + + + + builtins.typeOf + e + + Return a string representing the type of the value + e, namely "int", + "bool", "string", + "path", "null", + "set", "list", + "lambda" or + "float". + + + + + + + +
diff --git a/third_party/nix/doc/manual/expressions/derivations.xml b/third_party/nix/doc/manual/expressions/derivations.xml new file mode 100644 index 000000000..6f6297565 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/derivations.xml @@ -0,0 +1,211 @@ +
+ +Derivations + +The most important built-in function is +derivation, which is used to describe a single +derivation (a build action). It takes as input a set, the attributes +of which specify the inputs of the build. + + + + There must be an attribute named + system whose value must be a string specifying a + Nix platform identifier, such as "i686-linux" or + "x86_64-darwin"To figure out + your platform identifier, look at the line Checking for the + canonical Nix system name in the output of Nix's + configure script. The build + can only be performed on a machine and operating system matching the + platform identifier. (Nix can automatically forward builds for + other platforms by forwarding them to other machines; see .) + + There must be an attribute named + name whose value must be a string. This is used + as a symbolic name for the package by nix-env, + and it is appended to the output paths of the + derivation. + + There must be an attribute named + builder that identifies the program that is + executed to perform the build. It can be either a derivation or a + source (a local file reference, e.g., + ./builder.sh). + + Every attribute is passed as an environment variable + to the builder. Attribute values are translated to environment + variables as follows: + + + + Strings and numbers are just passed + verbatim. + + A path (e.g., + ../foo/sources.tar) causes the referenced + file to be copied to the store; its location in the store is put + in the environment variable. The idea is that all sources + should reside in the Nix store, since all inputs to a derivation + should reside in the Nix store. + + A derivation causes that + derivation to be built prior to the present derivation; its + default output path is put in the environment + variable. + + Lists of the previous types are also allowed. + They are simply concatenated, separated by + spaces. + + true is passed as the string + 1, false and + null are passed as an empty string. + + + + + + The optional attribute args + specifies command-line arguments to be passed to the builder. It + should be a list. + + The optional attribute outputs + specifies a list of symbolic outputs of the derivation. By default, + a derivation produces a single output path, denoted as + out. However, derivations can produce multiple + output paths. This is useful because it allows outputs to be + downloaded or garbage-collected separately. For instance, imagine a + library package that provides a dynamic library, header files, and + documentation. A program that links against the library doesn’t + need the header files and documentation at runtime, and it doesn’t + need the documentation at build time. Thus, the library package + could specify: + +outputs = [ "lib" "headers" "doc" ]; + + This will cause Nix to pass environment variables + lib, headers and + doc to the builder containing the intended store + paths of each output. The builder would typically do something like + +./configure --libdir=$lib/lib --includedir=$headers/include --docdir=$doc/share/doc + + for an Autoconf-style package. You can refer to each output of a + derivation by selecting it as an attribute, e.g. + +buildInputs = [ pkg.lib pkg.headers ]; + + The first element of outputs determines the + default output. Thus, you could also write + +buildInputs = [ pkg pkg.headers ]; + + since pkg is equivalent to + pkg.lib. + + + +The function mkDerivation in the Nixpkgs +standard environment is a wrapper around +derivation that adds a default value for +system and always uses Bash as the builder, to +which the supplied builder is passed as a command-line argument. See +the Nixpkgs manual for details. + +The builder is executed as follows: + + + + A temporary directory is created under the directory + specified by TMPDIR (default + /tmp) where the build will take place. The + current directory is changed to this directory. + + The environment is cleared and set to the derivation + attributes, as specified above. + + In addition, the following variables are set: + + + + NIX_BUILD_TOP contains the path of + the temporary directory for this build. + + Also, TMPDIR, + TEMPDIR, TMP, TEMP + are set to point to the temporary directory. This is to prevent + the builder from accidentally writing temporary files anywhere + else. Doing so might cause interference by other + processes. + + PATH is set to + /path-not-set to prevent shells from + initialising it to their built-in default value. + + HOME is set to + /homeless-shelter to prevent programs from + using /etc/passwd or the like to find the + user's home directory, which could cause impurity. Usually, when + HOME is set, it is used as the location of the home + directory, even if it points to a non-existent + path. + + NIX_STORE is set to the path of the + top-level Nix store directory (typically, + /nix/store). + + For each output declared in + outputs, the corresponding environment variable + is set to point to the intended path in the Nix store for that + output. Each output path is a concatenation of the cryptographic + hash of all build inputs, the name attribute + and the output name. (The output name is omitted if it’s + out.) + + + + + + If an output path already exists, it is removed. + Also, locks are acquired to prevent multiple Nix instances from + performing the same build at the same time. + + A log of the combined standard output and error is + written to /nix/var/log/nix. + + The builder is executed with the arguments specified + by the attribute args. If it exits with exit + code 0, it is considered to have succeeded. + + The temporary directory is removed (unless the + option was specified). + + If the build was successful, Nix scans each output + path for references to input paths by looking for the hash parts of + the input paths. Since these are potential runtime dependencies, + Nix registers them as dependencies of the output + paths. + + After the build, Nix sets the last-modified + timestamp on all files in the build result to 1 (00:00:01 1/1/1970 + UTC), sets the group to the default group, and sets the mode of the + file to 0444 or 0555 (i.e., read-only, with execute permission + enabled if the file was originally executable). Note that possible + setuid and setgid bits are + cleared. Setuid and setgid programs are not currently supported by + Nix. This is because the Nix archives used in deployment have no + concept of ownership information, and because it makes the build + result dependent on the user performing the build. + + + + + + + +
diff --git a/third_party/nix/doc/manual/expressions/expression-language.xml b/third_party/nix/doc/manual/expressions/expression-language.xml new file mode 100644 index 000000000..240ef80f1 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/expression-language.xml @@ -0,0 +1,30 @@ + + +Nix Expression Language + +The Nix expression language is a pure, lazy, functional +language. Purity means that operations in the language don't have +side-effects (for instance, there is no variable assignment). +Laziness means that arguments to functions are evaluated only when +they are needed. Functional means that functions are +normal values that can be passed around and manipulated +in interesting ways. The language is not a full-featured, general +purpose language. Its main job is to describe packages, +compositions of packages, and the variability within +packages. + +This section presents the various features of the +language. + + + + + + + + + diff --git a/third_party/nix/doc/manual/expressions/expression-syntax.xml b/third_party/nix/doc/manual/expressions/expression-syntax.xml new file mode 100644 index 000000000..42b9dca36 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/expression-syntax.xml @@ -0,0 +1,148 @@ +
+ +Expression Syntax + +Nix expression for GNU Hello +(<filename>default.nix</filename>) + +{ stdenv, fetchurl, perl }: + +stdenv.mkDerivation { + name = "hello-2.1.1"; + builder = ./builder.sh; + src = fetchurl { + url = ftp://ftp.nluug.nl/pub/gnu/hello/hello-2.1.1.tar.gz; + sha256 = "1md7jsfd8pa45z73bz1kszpp01yw6x5ljkjk2hx7wl800any6465"; + }; + inherit perl; +} + + + shows a Nix expression for GNU +Hello. It's actually already in the Nix Packages collection in +pkgs/applications/misc/hello/ex-1/default.nix. +It is customary to place each package in a separate directory and call +the single Nix expression in that directory +default.nix. The file has the following elements +(referenced from the figure by number): + + + + + + This states that the expression is a + function that expects to be called with three + arguments: stdenv, fetchurl, + and perl. They are needed to build Hello, but + we don't know how to build them here; that's why they are function + arguments. stdenv is a package that is used + by almost all Nix Packages packages; it provides a + standard environment consisting of the things you + would expect in a basic Unix environment: a C/C++ compiler (GCC, + to be precise), the Bash shell, fundamental Unix tools such as + cp, grep, + tar, etc. fetchurl is a + function that downloads files. perl is the + Perl interpreter. + + Nix functions generally have the form { x, y, ..., + z }: e where x, y, + etc. are the names of the expected arguments, and where + e is the body of the function. So + here, the entire remainder of the file is the body of the + function; when given the required arguments, the body should + describe how to build an instance of the Hello package. + + + + + + So we have to build a package. Building something from + other stuff is called a derivation in Nix (as + opposed to sources, which are built by humans instead of + computers). We perform a derivation by calling + stdenv.mkDerivation. + mkDerivation is a function provided by + stdenv that builds a package from a set of + attributes. A set is just a list of + key/value pairs where each key is a string and each value is an + arbitrary Nix expression. They take the general form { + name1 = + expr1; ... + nameN = + exprN; }. + + + + + + The attribute name specifies the symbolic + name and version of the package. Nix doesn't really care about + these things, but they are used by for instance nix-env + -q to show a human-readable name for + packages. This attribute is required by + mkDerivation. + + + + + + The attribute builder specifies the + builder. This attribute can sometimes be omitted, in which case + mkDerivation will fill in a default builder + (which does a configure; make; make install, in + essence). Hello is sufficiently simple that the default builder + would suffice, but in this case, we will show an actual builder + for educational purposes. The value + ./builder.sh refers to the shell script shown + in , discussed below. + + + + + + The builder has to know what the sources of the package + are. Here, the attribute src is bound to the + result of a call to the fetchurl function. + Given a URL and a SHA-256 hash of the expected contents of the file + at that URL, this function builds a derivation that downloads the + file and checks its hash. So the sources are a dependency that + like all other dependencies is built before Hello itself is + built. + + Instead of src any other name could have + been used, and in fact there can be any number of sources (bound + to different attributes). However, src is + customary, and it's also expected by the default builder (which we + don't use in this example). + + + + + + Since the derivation requires Perl, we have to pass the + value of the perl function argument to the + builder. All attributes in the set are actually passed as + environment variables to the builder, so declaring an attribute + + +perl = perl; + + will do the trick: it binds an attribute perl + to the function argument which also happens to be called + perl. However, it looks a bit silly, so there + is a shorter syntax. The inherit keyword + causes the specified attributes to be bound to whatever variables + with the same name happen to be in scope. + + + + + + + +
diff --git a/third_party/nix/doc/manual/expressions/generic-builder.xml b/third_party/nix/doc/manual/expressions/generic-builder.xml new file mode 100644 index 000000000..db7ff405d --- /dev/null +++ b/third_party/nix/doc/manual/expressions/generic-builder.xml @@ -0,0 +1,98 @@ +
+ +Generic Builder Syntax + +Recall from that the builder +looked something like this: + + +PATH=$perl/bin:$PATH +tar xvfz $src +cd hello-* +./configure --prefix=$out +make +make install + +The builders for almost all Unix packages look like this — set up some +environment variables, unpack the sources, configure, build, and +install. For this reason the standard environment provides some Bash +functions that automate the build process. A builder using the +generic build facilities in shown in . + +Build script using the generic +build functions + +buildInputs="$perl" + +source $stdenv/setup + +genericBuild + + + + + + + The buildInputs variable tells + setup to use the indicated packages as + inputs. This means that if a package provides a + bin subdirectory, it's added to + PATH; if it has a include + subdirectory, it's added to GCC's header search path; and so + on.How does it work? setup + tries to source the file + pkg/nix-support/setup-hook + of all dependencies. These “setup hooks” can then set up whatever + environment variables they want; for instance, the setup hook for + Perl sets the PERL5LIB environment variable to + contain the lib/site_perl directories of all + inputs. + + + + + + + The function genericBuild is defined in + the file $stdenv/setup. + + + + + + The final step calls the shell function + genericBuild, which performs the steps that + were done explicitly in . The + generic builder is smart enough to figure out whether to unpack + the sources using gzip, + bzip2, etc. It can be customised in many ways; + see the Nixpkgs manual for details. + + + + + +Discerning readers will note that the +buildInputs could just as well have been set in the Nix +expression, like this: + + + buildInputs = [ perl ]; + +The perl attribute can then be removed, and the +builder becomes even shorter: + + +source $stdenv/setup +genericBuild + +In fact, mkDerivation provides a default builder +that looks exactly like that, so it is actually possible to omit the +builder for Hello entirely. + +
diff --git a/third_party/nix/doc/manual/expressions/language-constructs.xml b/third_party/nix/doc/manual/expressions/language-constructs.xml new file mode 100644 index 000000000..0d0cbbe15 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/language-constructs.xml @@ -0,0 +1,409 @@ +
+ +Language Constructs + +Recursive sets + +Recursive sets are just normal sets, but the attributes can +refer to each other. For example, + + +rec { + x = y; + y = 123; +}.x + + +evaluates to 123. Note that without +rec the binding x = y; would +refer to the variable y in the surrounding scope, +if one exists, and would be invalid if no such variable exists. That +is, in a normal (non-recursive) set, attributes are not added to the +lexical scope; in a recursive set, they are. + +Recursive sets of course introduce the danger of infinite +recursion. For example, + + +rec { + x = y; + y = x; +}.x + +does not terminateActually, Nix detects infinite +recursion in this case and aborts (infinite recursion +encountered).. + + + + +Let-expressions + +A let-expression allows you to define local variables for an +expression. For instance, + + +let + x = "foo"; + y = "bar"; +in x + y + +evaluates to "foobar". + + + + + + +Inheriting attributes + +When defining a set or in a let-expression it is often convenient to copy variables +from the surrounding lexical scope (e.g., when you want to propagate +attributes). This can be shortened using the +inherit keyword. For instance, + + +let x = 123; in +{ inherit x; + y = 456; +} + +is equivalent to + + +let x = 123; in +{ x = x; + y = 456; +} + +and both evaluate to { x = 123; y = 456; }. (Note that +this works because x is added to the lexical scope +by the let construct.) It is also possible to +inherit attributes from another set. For instance, in this fragment +from all-packages.nix, + + + graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xlibs) libXaw; + }; + + xlibs = { + libX11 = ...; + libXaw = ...; + ... + } + + libpng = ...; + libjpg = ...; + ... + +the set used in the function call to the function defined in +../tools/graphics/graphviz inherits a number of +variables from the surrounding scope (fetchurl +... yacc), but also inherits +libXaw (the X Athena Widgets) from the +xlibs (X11 client-side libraries) set. + + +Summarizing the fragment + + +... +inherit x y z; +inherit (src-set) a b c; +... + +is equivalent to + + +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... + +when used while defining local variables in a let-expression or +while defining a set. + + + + +Functions + +Functions have the following form: + + +pattern: body + +The pattern specifies what the argument of the function must look +like, and binds variables in the body to (parts of) the +argument. There are three kinds of patterns: + + + + + If a pattern is a single identifier, then the + function matches any argument. Example: + + +let negate = x: !x; + concat = x: y: x + y; +in if negate true then concat "foo" "bar" else "" + + Note that concat is a function that takes one + argument and returns a function that takes another argument. This + allows partial parameterisation (i.e., only filling some of the + arguments of a function); e.g., + + +map (concat "foo") [ "bar" "bla" "abc" ] + + evaluates to [ "foobar" "foobla" + "fooabc" ]. + + + A set pattern of the form + { name1, name2, …, nameN } matches a set + containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + +{ x, y, z }: z + y + x + + can only be called with a set containing exactly the attributes + x, y and + z. No other attributes are allowed. If you want + to allow additional arguments, you can use an ellipsis + (...): + + +{ x, y, z, ... }: z + y + x + + This works on any set that contains at least the three named + attributes. + + It is possible to provide default values + for attributes, in which case they are allowed to be missing. A + default value is specified by writing + name ? + e, where + e is an arbitrary expression. For example, + + +{ x, y ? "foo", z ? "bar" }: z + y + x + + specifies a function that only requires an attribute named + x, but optionally accepts y + and z. + + + An @-pattern provides a means of referring + to the whole value being matched: + + args@{ x, y, z, ... }: z + y + x + args.a + +but can also be written as: + + { x, y, z, ... } @ args: z + y + x + args.a + + Here args is bound to the entire argument, which + is further matched against the pattern { x, y, z, + ... }. @-pattern makes mainly sense with an + ellipsis(...) as you can access attribute names as + a, using args.a, which was given as an + additional attribute to the function. + + + + + The args@ expression is bound to the argument passed to the function which + means that attributes with defaults that aren't explicitly specified in the function call + won't cause an evaluation error, but won't exist in args. + + + For instance + +let + function = args@{ a ? 23, ... }: args; +in + function {} + + will evaluate to an empty attribute set. + + + + + +Note that functions do not have names. If you want to give them +a name, you can bind them to an attribute, e.g., + + +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } + + + + + + +Conditionals + +Conditionals look like this: + + +if e1 then e2 else e3 + +where e1 is an expression that should +evaluate to a Boolean value (true or +false). + + + + +Assertions + +Assertions are generally used to check that certain requirements +on or between features and dependencies hold. They look like this: + + +assert e1; e2 + +where e1 is an expression that should +evaluate to a Boolean value. If it evaluates to +true, e2 is returned; +otherwise expression evaluation is aborted and a backtrace is printed. + +Nix expression for Subversion + +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; +assert httpServer -> httpd != null && httpd.expat == expat; +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; + ... +} + + + show how assertions are +used in the Nix expression for Subversion. + + + + + This assertion states that if Subversion is to have support + for local repositories, then Berkeley DB is needed. So if the + Subversion function is called with the + localServer argument set to + true but the db4 argument + set to null, then the evaluation fails. + + + + This is a more subtle condition: if Subversion is built with + Apache (httpServer) support, then the Expat + library (an XML library) used by Subversion should be same as the + one used by Apache. This is because in this configuration + Subversion code ends up being linked with Apache code, and if the + Expat libraries do not match, a build- or runtime link error or + incompatibility might occur. + + + + This assertion says that in order for Subversion to have SSL + support (so that it can access https URLs), an + OpenSSL library must be passed. Additionally, it says that + if Apache support is enabled, then Apache's + OpenSSL should match Subversion's. (Note that if Apache support + is not enabled, we don't care about Apache's OpenSSL.) + + + + The conditional here is not really related to assertions, + but is worth pointing out: it ensures that if SSL support is + disabled, then the Subversion derivation is not dependent on + OpenSSL, even if a non-null value was passed. + This prevents an unnecessary rebuild of Subversion if OpenSSL + changes. + + + + + + + + +With-expressions + +A with-expression, + + +with e1; e2 + +introduces the set e1 into the lexical +scope of the expression e2. For instance, + + +let as = { x = "foo"; y = "bar"; }; +in with as; x + y + +evaluates to "foobar" since the +with adds the x and +y attributes of as to the +lexical scope in the expression x + y. The most +common use of with is in conjunction with the +import function. E.g., + + +with (import ./definitions.nix); ... + +makes all attributes defined in the file +definitions.nix available as if they were defined +locally in a let-expression. + +The bindings introduced by with do not shadow bindings +introduced by other means, e.g. + + +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... + +establishes the same scope as + + +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... + + + + + + +Comments + +Comments can be single-line, started with a # +character, or inline/multi-line, enclosed within /* +... */. + + + + +
diff --git a/third_party/nix/doc/manual/expressions/language-operators.xml b/third_party/nix/doc/manual/expressions/language-operators.xml new file mode 100644 index 000000000..4f11bf529 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/language-operators.xml @@ -0,0 +1,222 @@ +
+ +Operators + + lists the operators in the +Nix expression language, in order of precedence (from strongest to +weakest binding). + + + Operators + + + + Name + Syntax + Associativity + Description + Precedence + + + + + Select + e . + attrpath + [ or def ] + + none + Select attribute denoted by the attribute path + attrpath from set + e. (An attribute path is a + dot-separated list of attribute names.) If the attribute + doesn’t exist, return def if + provided, otherwise abort evaluation. + 1 + + + Application + e1 e2 + left + Call function e1 with + argument e2. + 2 + + + Arithmetic Negation + - e + none + Arithmetic negation. + 3 + + + Has Attribute + e ? + attrpath + none + Test whether set e contains + the attribute denoted by attrpath; + return true or + false. + 4 + + + List Concatenation + e1 ++ e2 + right + List concatenation. + 5 + + + Multiplication + + e1 * e2, + + left + Arithmetic multiplication. + 6 + + + Division + + e1 / e2 + + left + Arithmetic division. + 6 + + + Addition + + e1 + e2 + + left + Arithmetic addition. + 7 + + + Subtraction + + e1 - e2 + + left + Arithmetic subtraction. + 7 + + + String Concatenation + + string1 + string2 + + left + String concatenation. + 7 + + + Not + ! e + none + Boolean negation. + 8 + + + Update + e1 // + e2 + right + Return a set consisting of the attributes in + e1 and + e2 (with the latter taking + precedence over the former in case of equally named + attributes). + 9 + + + Less Than + + e1 < e2, + + none + Arithmetic comparison. + 10 + + + Less Than or Equal To + + e1 <= e2 + + none + Arithmetic comparison. + 10 + + + Greater Than + + e1 > e2 + + none + Arithmetic comparison. + 10 + + + Greater Than or Equal To + + e1 >= e2 + + none + Arithmetic comparison. + 10 + + + Equality + + e1 == e2 + + none + Equality. + 11 + + + Inequality + + e1 != e2 + + none + Inequality. + 11 + + + Logical AND + e1 && + e2 + left + Logical AND. + 12 + + + Logical OR + e1 || + e2 + left + Logical OR. + 13 + + + Logical Implication + e1 -> + e2 + none + Logical implication (equivalent to + !e1 || + e2). + 14 + + + +
+ +
diff --git a/third_party/nix/doc/manual/expressions/language-values.xml b/third_party/nix/doc/manual/expressions/language-values.xml new file mode 100644 index 000000000..bb2090c88 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/language-values.xml @@ -0,0 +1,313 @@ +
+ +Values + + +Simple Values + +Nix has the following basic data types: + + + + + + Strings can be written in three + ways. + + The most common way is to enclose the string between double + quotes, e.g., "foo bar". Strings can span + multiple lines. The special characters " and + \ and the character sequence + ${ must be escaped by prefixing them with a + backslash (\). Newlines, carriage returns and + tabs can be written as \n, + \r and \t, + respectively. + + You can include the result of an expression into a string by + enclosing it in + ${...}, a feature + known as antiquotation. The enclosed + expression must evaluate to something that can be coerced into a + string (meaning that it must be a string, a path, or a + derivation). For instance, rather than writing + + +"--with-freetype2-library=" + freetype + "/lib" + + (where freetype is a derivation), you can + instead write the more natural + + +"--with-freetype2-library=${freetype}/lib" + + The latter is automatically translated to the former. A more + complicated example (from the Nix expression for Qt): + + +configureFlags = " + -system-zlib -system-libpng -system-libjpeg + ${if openglSupport then "-dlopen-opengl + -L${mesa}/lib -I${mesa}/include + -L${libXmu}/lib -I${libXmu}/include" else ""} + ${if threadSupport then "-thread" else "-no-thread"} +"; + + Note that Nix expressions and strings can be arbitrarily nested; + in this case the outer string contains various antiquotations that + themselves contain strings (e.g., "-thread"), + some of which in turn contain expressions (e.g., + ${mesa}). + + The second way to write string literals is as an + indented string, which is enclosed between + pairs of double single-quotes, like so: + + +'' + This is the first line. + This is the second line. + This is the third line. +'' + + This kind of string literal intelligently strips indentation from + the start of each line. To be precise, it strips from each line a + number of spaces equal to the minimal indentation of the string as + a whole (disregarding the indentation of empty lines). For + instance, the first and second line are indented two space, while + the third line is indented four spaces. Thus, two spaces are + stripped from each line, so the resulting string is + + +"This is the first line.\nThis is the second line.\n This is the third line.\n" + + + + Note that the whitespace and newline following the opening + '' is ignored if there is no non-whitespace + text on the initial line. + + Antiquotation + (${expr}) is + supported in indented strings. + + Since ${ and '' have + special meaning in indented strings, you need a way to quote them. + $ can be escaped by prefixing it with + '' (that is, two single quotes), i.e., + ''$. '' can be escaped by + prefixing it with ', i.e., + '''. $ removes any special meaning + from the following $. Linefeed, carriage-return and tab + characters can be written as ''\n, + ''\r, ''\t, and ''\ + escapes any other character. + + + + Indented strings are primarily useful in that they allow + multi-line string literals to follow the indentation of the + enclosing Nix expression, and that less escaping is typically + necessary for strings representing languages such as shell scripts + and configuration files because '' is much less + common than ". Example: + + +stdenv.mkDerivation { + ... + postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; + ... +} + + + + + Finally, as a convenience, URIs as + defined in appendix B of RFC 2396 + can be written as is, without quotes. For + instance, the string + "http://example.org/foo.tar.bz2" + can also be written as + http://example.org/foo.tar.bz2. + + + + Numbers, which can be integers (like + 123) or floating point (like + 123.43 or .27e13). + + Numbers are type-compatible: pure integer operations will always + return integers, whereas any operation involving at least one floating point + number will have a floating point number as a result. + + Paths, e.g., + /bin/sh or ./builder.sh. + A path must contain at least one slash to be recognised as such; for + instance, builder.sh is not a + pathIt's parsed as an expression that selects the + attribute sh from the variable + builder.. If the file name is + relative, i.e., if it does not begin with a slash, it is made + absolute at parse time relative to the directory of the Nix + expression that contained it. For instance, if a Nix expression in + /foo/bar/bla.nix refers to + ../xyzzy/fnord.nix, the absolute path is + /foo/xyzzy/fnord.nix. + + If the first component of a path is a ~, + it is interpreted as if the rest of the path were relative to the + user's home directory. e.g. ~/foo would be + equivalent to /home/edolstra/foo for a user + whose home directory is /home/edolstra. + + + Paths can also be specified between angle brackets, e.g. + <nixpkgs>. This means that the directories + listed in the environment variable + NIX_PATH will be searched + for the given file or directory name. + + + + + Booleans with values + true and + false. + + The null value, denoted as + null. + + + + + + + + +Lists + +Lists are formed by enclosing a whitespace-separated list of +values between square brackets. For example, + + +[ 123 ./foo.nix "abc" (f { x = y; }) ] + +defines a list of four elements, the last being the result of a call +to the function f. Note that function calls have +to be enclosed in parentheses. If they had been omitted, e.g., + + +[ 123 ./foo.nix "abc" f { x = y; } ] + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + + + + + +Sets + +Sets are really the core of the language, since ultimately the +Nix language is all about creating derivations, which are really just +sets of attributes to be passed to build scripts. + +Sets are just a list of name/value pairs (called +attributes) enclosed in curly brackets, where +each value is an arbitrary expression terminated by a semicolon. For +example: + + +{ x = 123; + text = "Hello"; + y = f { bla = 456; }; +} + +This defines a set with attributes named x, +text, y. The order of the +attributes is irrelevant. An attribute name may only occur +once. + +Attributes can be selected from a set using the +. operator. For instance, + + +{ a = "Foo"; b = "Bar"; }.a + +evaluates to "Foo". It is possible to provide a +default value in an attribute selection using the +or keyword. For example, + + +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" + +will evaluate to "Xyzzy" because there is no +c attribute in the set. + +You can use arbitrary double-quoted strings as attribute +names: + + +{ "foo ${bar}" = 123; "nix-1.0" = 456; }."foo ${bar}" + + +This will evaluate to 123 (Assuming +bar is antiquotable). In the case where an +attribute name is just a single antiquotation, the quotes can be +dropped: + + +{ foo = 123; }.${bar} or 456 + +This will evaluate to 123 if +bar evaluates to "foo" when +coerced to a string and 456 otherwise (again +assuming bar is antiquotable). + +In the special case where an attribute name inside of a set declaration +evaluates to null (which is normally an error, as +null is not antiquotable), that attribute is simply not +added to the set: + + +{ ${if foo then "bar" else null} = true; } + +This will evaluate to {} if foo +evaluates to false. + +A set that has a __functor attribute whose value +is callable (i.e. is itself a function or a set with a +__functor attribute whose value is callable) can be +applied as if it were a function, with the set itself passed in first +, e.g., + + +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 + + +evaluates to 2. This can be used to attach metadata to a +function without the caller needing to treat it specially, or to implement +a form of object-oriented programming, for example. + + + + + + +
diff --git a/third_party/nix/doc/manual/expressions/simple-building-testing.xml b/third_party/nix/doc/manual/expressions/simple-building-testing.xml new file mode 100644 index 000000000..7326a3e76 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/simple-building-testing.xml @@ -0,0 +1,84 @@ +
+ +Building and Testing + +You can now try to build Hello. Of course, you could do +nix-env -i hello, but you may not want to install a +possibly broken package just yet. The best way to test the package is by +using the command nix-build, +which builds a Nix expression and creates a symlink named +result in the current directory: + + +$ nix-build -A hello +building path `/nix/store/632d2b22514d...-hello-2.1.1' +hello-2.1.1/ +hello-2.1.1/intl/ +hello-2.1.1/intl/ChangeLog +... + +$ ls -l result +lrwxrwxrwx ... 2006-09-29 10:43 result -> /nix/store/632d2b22514d...-hello-2.1.1 + +$ ./result/bin/hello +Hello, world! + +The option selects +the hello attribute. This is faster than using the +symbolic package name specified by the name +attribute (which also happens to be hello) and is +unambiguous (there can be multiple packages with the symbolic name +hello, but there can be only one attribute in a set +named hello). + +nix-build registers the +./result symlink as a garbage collection root, so +unless and until you delete the ./result symlink, +the output of the build will be safely kept on your system. You can +use nix-build’s switch to give the symlink another +name. + +Nix has transactional semantics. Once a build finishes +successfully, Nix makes a note of this in its database: it registers +that the path denoted by out is now +valid. If you try to build the derivation again, Nix +will see that the path is already valid and finish immediately. If a +build fails, either because it returns a non-zero exit code, because +Nix or the builder are killed, or because the machine crashes, then +the output paths will not be registered as valid. If you try to build +the derivation again, Nix will remove the output paths if they exist +(e.g., because the builder died half-way through make +install) and try again. Note that there is no +negative caching: Nix doesn't remember that a build +failed, and so a failed build can always be repeated. This is because +Nix cannot distinguish between permanent failures (e.g., a compiler +error due to a syntax error in the source) and transient failures +(e.g., a disk full condition). + +Nix also performs locking. If you run multiple Nix builds +simultaneously, and they try to build the same derivation, the first +Nix instance that gets there will perform the build, while the others +block (or perform other derivations if available) until the build +finishes: + + +$ nix-build -A hello +waiting for lock on `/nix/store/0h5b7hp8d4hqfrw8igvx97x1xawrjnac-hello-2.1.1x' + +So it is always safe to run multiple instances of Nix in parallel +(which isn’t the case with, say, make). + +If you have a system with multiple CPUs, you may want to have +Nix build different derivations in parallel (insofar as possible). +Just pass the option , where +N is the maximum number of jobs to be run +in parallel, or set. Typically this should be the number of +CPUs. + +
diff --git a/third_party/nix/doc/manual/expressions/simple-expression.xml b/third_party/nix/doc/manual/expressions/simple-expression.xml new file mode 100644 index 000000000..29fd872ee --- /dev/null +++ b/third_party/nix/doc/manual/expressions/simple-expression.xml @@ -0,0 +1,47 @@ + + +A Simple Nix Expression + +This section shows how to add and test the GNU Hello +package to the Nix Packages collection. Hello is a program +that prints out the text Hello, world!. + +To add a package to the Nix Packages collection, you generally +need to do three things: + + + + Write a Nix expression for the package. This is a + file that describes all the inputs involved in building the package, + such as dependencies, sources, and so on. + + Write a builder. This is a + shell scriptIn fact, it can be written in any + language, but typically it's a bash shell + script. that actually builds the package from + the inputs. + + Add the package to the file + pkgs/top-level/all-packages.nix. The Nix + expression written in the first step is a + function; it requires other packages in order + to build it. In this step you put it all together, i.e., you call + the function with the right arguments to build the actual + package. + + + + + + + + + + + + diff --git a/third_party/nix/doc/manual/expressions/writing-nix-expressions.xml b/third_party/nix/doc/manual/expressions/writing-nix-expressions.xml new file mode 100644 index 000000000..6646dddf0 --- /dev/null +++ b/third_party/nix/doc/manual/expressions/writing-nix-expressions.xml @@ -0,0 +1,26 @@ + + +Writing Nix Expressions + + +This chapter shows you how to write Nix expressions, which +instruct Nix how to build packages. It starts with a +simple example (a Nix expression for GNU Hello), and then moves +on to a more in-depth look at the Nix expression language. + +This chapter is mostly about the Nix expression language. +For more extensive information on adding packages to the Nix Packages +collection (such as functions in the standard environment and coding +conventions), please consult its +manual. + + + + + + diff --git a/third_party/nix/doc/manual/figures/user-environments.png b/third_party/nix/doc/manual/figures/user-environments.png new file mode 100644 index 000000000..1f781cf23 Binary files /dev/null and b/third_party/nix/doc/manual/figures/user-environments.png differ diff --git a/third_party/nix/doc/manual/figures/user-environments.sxd b/third_party/nix/doc/manual/figures/user-environments.sxd new file mode 100644 index 000000000..bc661b640 Binary files /dev/null and b/third_party/nix/doc/manual/figures/user-environments.sxd differ diff --git a/third_party/nix/doc/manual/glossary/glossary.xml b/third_party/nix/doc/manual/glossary/glossary.xml new file mode 100644 index 000000000..e3162ed8d --- /dev/null +++ b/third_party/nix/doc/manual/glossary/glossary.xml @@ -0,0 +1,199 @@ + + +Glossary + + + + + +derivation + + A description of a build action. The result of a + derivation is a store object. Derivations are typically specified + in Nix expressions using the derivation + primitive. These are translated into low-level + store derivations (implicitly by + nix-env and nix-build, or + explicitly by nix-instantiate). + + + + +store + + The location in the file system where store objects + live. Typically /nix/store. + + + + +store path + + The location in the file system of a store object, + i.e., an immediate child of the Nix store + directory. + + + + +store object + + A file that is an immediate child of the Nix store + directory. These can be regular files, but also entire directory + trees. Store objects can be sources (objects copied from outside of + the store), derivation outputs (objects produced by running a build + action), or derivations (files describing a build + action). + + + + +substitute + + A substitute is a command invocation stored in the + Nix database that describes how to build a store object, bypassing + the normal build mechanism (i.e., derivations). Typically, the + substitute builds the store object by downloading a pre-built + version of the store object from some server. + + + + +purity + + The assumption that equal Nix derivations when run + always produce the same output. This cannot be guaranteed in + general (e.g., a builder can rely on external inputs such as the + network or the system time) but the Nix model assumes + it. + + + + +Nix expression + + A high-level description of software packages and + compositions thereof. Deploying software using Nix entails writing + Nix expressions for your packages. Nix expressions are translated + to derivations that are stored in the Nix store. These derivations + can then be built. + + + + +reference + + + A store path P is said to have a + reference to a store path Q if the store object + at P contains the path Q + somewhere. The references of a store path are + the set of store paths to which it has a reference. + + A derivation can reference other derivations and sources + (but not output paths), whereas an output path only references other + output paths. + + + + + +reachable + + A store path Q is reachable from + another store path P if Q is in the + closure of the + references relation. + + + +closure + + The closure of a store path is the set of store + paths that are directly or indirectly “reachable” from that store + path; that is, it’s the closure of the path under the references relation. For a package, the + closure of its derivation is equivalent to the build-time + dependencies, while the closure of its output path is equivalent to its + runtime dependencies. For correct deployment it is necessary to deploy whole + closures, since otherwise at runtime files could be missing. The command + nix-store -qR prints out closures of store paths. + + As an example, if the store object at path P contains + a reference to path Q, then Q is + in the closure of P. Further, if Q + references R then R is also in + the closure of P. + + + + + +output path + + A store path produced by a derivation. + + + + +deriver + + The deriver of an output path is the store + derivation that built it. + + + + +validity + + A store path is considered + valid if it exists in the file system, is + listed in the Nix database as being valid, and if all paths in its + closure are also valid. + + + + +user environment + + An automatically generated store object that + consists of a set of symlinks to “active” applications, i.e., other + store paths. These are generated automatically by nix-env. See . + + + + + + +profile + + A symlink to the current user environment of a user, e.g., + /nix/var/nix/profiles/default. + + + + +NAR + + A Nix + ARchive. This is a serialisation of a path in + the Nix store. It can contain regular files, directories and + symbolic links. NARs are generated and unpacked using + nix-store --dump and nix-store + --restore. + + + + + + + + + diff --git a/third_party/nix/doc/manual/hacking.xml b/third_party/nix/doc/manual/hacking.xml new file mode 100644 index 000000000..b671811d3 --- /dev/null +++ b/third_party/nix/doc/manual/hacking.xml @@ -0,0 +1,41 @@ + + +Hacking + +This section provides some notes on how to hack on Nix. To get +the latest version of Nix from GitHub: + +$ git clone git://github.com/NixOS/nix.git +$ cd nix + + + +To build it and its dependencies: + +$ nix-build release.nix -A build.x86_64-linux + + + +To build all dependencies and start a shell in which all +environment variables are set up so that those dependencies can be +found: + +$ nix-shell + +To build Nix itself in this shell: + +[nix-shell]$ ./bootstrap.sh +[nix-shell]$ configurePhase +[nix-shell]$ make + +To install it in $(pwd)/inst and test it: + +[nix-shell]$ make install +[nix-shell]$ make installcheck + + + + + diff --git a/third_party/nix/doc/manual/images/callouts/1.gif b/third_party/nix/doc/manual/images/callouts/1.gif new file mode 100644 index 000000000..9e7a87f75 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/1.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/10.gif b/third_party/nix/doc/manual/images/callouts/10.gif new file mode 100644 index 000000000..e80f7f8e6 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/10.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/11.gif b/third_party/nix/doc/manual/images/callouts/11.gif new file mode 100644 index 000000000..67f91a239 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/11.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/12.gif b/third_party/nix/doc/manual/images/callouts/12.gif new file mode 100644 index 000000000..54c4b42f1 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/12.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/13.gif b/third_party/nix/doc/manual/images/callouts/13.gif new file mode 100644 index 000000000..dd5d7d9b6 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/13.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/14.gif b/third_party/nix/doc/manual/images/callouts/14.gif new file mode 100644 index 000000000..3d7a952a3 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/14.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/15.gif b/third_party/nix/doc/manual/images/callouts/15.gif new file mode 100644 index 000000000..1c9183d5b Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/15.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/2.gif b/third_party/nix/doc/manual/images/callouts/2.gif new file mode 100644 index 000000000..94d42a30f Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/2.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/3.gif b/third_party/nix/doc/manual/images/callouts/3.gif new file mode 100644 index 000000000..dd3541a1b Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/3.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/4.gif b/third_party/nix/doc/manual/images/callouts/4.gif new file mode 100644 index 000000000..4bcbf7e31 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/4.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/5.gif b/third_party/nix/doc/manual/images/callouts/5.gif new file mode 100644 index 000000000..1c62b4f92 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/5.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/6.gif b/third_party/nix/doc/manual/images/callouts/6.gif new file mode 100644 index 000000000..23bc5555d Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/6.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/7.gif b/third_party/nix/doc/manual/images/callouts/7.gif new file mode 100644 index 000000000..e55ce8958 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/7.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/8.gif b/third_party/nix/doc/manual/images/callouts/8.gif new file mode 100644 index 000000000..49375e09f Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/8.gif differ diff --git a/third_party/nix/doc/manual/images/callouts/9.gif b/third_party/nix/doc/manual/images/callouts/9.gif new file mode 100644 index 000000000..da12a4fe2 Binary files /dev/null and b/third_party/nix/doc/manual/images/callouts/9.gif differ diff --git a/third_party/nix/doc/manual/installation/building-source.xml b/third_party/nix/doc/manual/installation/building-source.xml new file mode 100644 index 000000000..772cda9cc --- /dev/null +++ b/third_party/nix/doc/manual/installation/building-source.xml @@ -0,0 +1,49 @@ +
+ +Building Nix from Source + +After unpacking or checking out the Nix sources, issue the +following commands: + + +$ ./configure options... +$ make +$ make install + +Nix requires GNU Make so you may need to invoke +gmake instead. + +When building from the Git repository, these should be preceded +by the command: + + +$ ./bootstrap.sh + + + +The installation path can be specified by passing the + to +configure. The default installation directory is +/usr/local. You can change this to any location +you like. You must have write permission to the +prefix path. + +Nix keeps its store (the place where +packages are stored) in /nix/store by default. +This can be changed using +. + +It is best not to change the Nix +store from its default, since doing so makes it impossible to use +pre-built binaries from the standard Nixpkgs channels — that is, all +packages will need to be built from source. + +Nix keeps state (such as its database and log files) in +/nix/var by default. This can be changed using +. + +
diff --git a/third_party/nix/doc/manual/installation/env-variables.xml b/third_party/nix/doc/manual/installation/env-variables.xml new file mode 100644 index 000000000..e2b8fc867 --- /dev/null +++ b/third_party/nix/doc/manual/installation/env-variables.xml @@ -0,0 +1,89 @@ + + +Environment Variables + +To use Nix, some environment variables should be set. In +particular, PATH should contain the directories +prefix/bin and +~/.nix-profile/bin. The first directory contains +the Nix tools themselves, while ~/.nix-profile is +a symbolic link to the current user environment +(an automatically generated package consisting of symlinks to +installed packages). The simplest way to set the required environment +variables is to include the file +prefix/etc/profile.d/nix.sh +in your ~/.profile (or similar), like this: + + +source prefix/etc/profile.d/nix.sh + +
+ +<envar>NIX_SSL_CERT_FILE</envar> + +If you need to specify a custom certificate bundle to account +for an HTTPS-intercepting man in the middle proxy, you must specify +the path to the certificate bundle in the environment variable +NIX_SSL_CERT_FILE. + + +If you don't specify a NIX_SSL_CERT_FILE +manually, Nix will install and use its own certificate +bundle. + + + Set the environment variable and install Nix + +$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt +$ sh <(curl https://nixos.org/nix/install) + + + In the shell profile and rc files (for example, + /etc/bashrc, /etc/zshrc), + add the following line: + +export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt + + + + +You must not add the export and then do the install, as +the Nix installer will detect the presense of Nix configuration, and +abort. + +
+<envar>NIX_SSL_CERT_FILE</envar> with macOS and the Nix daemon + +On macOS you must specify the environment variable for the Nix +daemon service, then restart it: + + +$ sudo launchctl setenv NIX_SSL_CERT_FILE /etc/ssl/my-certificate-bundle.crt +$ sudo launchctl kickstart -k system/org.nixos.nix-daemon + +
+ +
+ +Proxy Environment Variables + +The Nix installer has special handling for these proxy-related +environment variables: +http_proxy, https_proxy, +ftp_proxy, no_proxy, +HTTP_PROXY, HTTPS_PROXY, +FTP_PROXY, NO_PROXY. + +If any of these variables are set when running the Nix installer, +then the installer will create an override file at +/etc/systemd/system/nix-daemon.service.d/override.conf +so nix-daemon will use them. + +
+ +
+ diff --git a/third_party/nix/doc/manual/installation/installation.xml b/third_party/nix/doc/manual/installation/installation.xml new file mode 100644 index 000000000..878959352 --- /dev/null +++ b/third_party/nix/doc/manual/installation/installation.xml @@ -0,0 +1,34 @@ + + +Installation + + +This section describes how to install and configure Nix for first-time use. + + + + + + + + + + + diff --git a/third_party/nix/doc/manual/installation/installing-binary.xml b/third_party/nix/doc/manual/installation/installing-binary.xml new file mode 100644 index 000000000..394d8053b --- /dev/null +++ b/third_party/nix/doc/manual/installation/installing-binary.xml @@ -0,0 +1,190 @@ + + +Installing a Binary Distribution + +If you are using Linux or macOS, the easiest way to install Nix +is to run the following command: + + + $ sh <(curl https://nixos.org/nix/install) + + +As of Nix 2.1.0, the Nix installer will always default to creating a +single-user installation, however opting in to the multi-user +installation is highly recommended. + + +
+ Single User Installation + + + To explicitly select a single-user installation on your system: + + + sh <(curl https://nixos.org/nix/install) --no-daemon + + + + +This will perform a single-user installation of Nix, meaning that +/nix is owned by the invoking user. You should +run this under your usual user account, not as +root. The script will invoke sudo to create +/nix if it doesn’t already exist. If you don’t +have sudo, you should manually create +/nix first as root, e.g.: + + +$ mkdir /nix +$ chown alice /nix + + +The install script will modify the first writable file from amongst +.bash_profile, .bash_login +and .profile to source +~/.nix-profile/etc/profile.d/nix.sh. You can set +the NIX_INSTALLER_NO_MODIFY_PROFILE environment +variable before executing the install script to disable this +behaviour. + + + +You can uninstall Nix simply by running: + + +$ rm -rf /nix + + + +
+ +
+ Multi User Installation + + The multi-user Nix installation creates system users, and a system + service for the Nix daemon. + + + + Supported Systems + + + Linux running systemd, with SELinux disabled + + macOS + + + + You can instruct the installer to perform a multi-user + installation on your system: + + + sh <(curl https://nixos.org/nix/install) --daemon + + + + + The multi-user installation of Nix will create build users between + the user IDs 30001 and 30032, and a group with the group ID 30000. + + You should run this under your usual user account, + not as root. The script will invoke + sudo as needed. + + + + If you need Nix to use a different group ID or user ID set, you + will have to download the tarball manually and edit the install + script. + + + + The installer will modify /etc/bashrc, and + /etc/zshrc if they exist. The installer will + first back up these files with a + .backup-before-nix extension. The installer + will also create /etc/profile.d/nix.sh. + + + You can uninstall Nix with the following commands: + + +sudo rm -rf /etc/profile/nix.sh /etc/nix /nix ~root/.nix-profile ~root/.nix-defexpr ~root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels + +# If you are on Linux with systemd, you will need to run: +sudo systemctl stop nix-daemon.socket +sudo systemctl stop nix-daemon.service +sudo systemctl disable nix-daemon.socket +sudo systemctl disable nix-daemon.service +sudo systemctl daemon-reload + +# If you are on macOS, you will need to run: +sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist +sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist + + + There may also be references to Nix in + /etc/profile, + /etc/bashrc, and + /etc/zshrc which you may remove. + + +
+ +
+ Installing a pinned Nix version from a URL + + + NixOS.org hosts version-specific installation URLs for all Nix + versions since 1.11.16, at + https://nixos.org/releases/nix/nix-VERSION/install. + + + + These install scripts can be used the same as the main + NixOS.org installation script: + + + sh <(curl https://nixos.org/nix/install) + + + + + In the same directory of the install script are sha256 sums, and + gpg signature files. + +
+ +
+ Installing from a binary tarball + + + You can also download a binary tarball that contains Nix and all + its dependencies. (This is what the install script at + https://nixos.org/nix/install does automatically.) You + should unpack it somewhere (e.g. in /tmp), + and then run the script named install inside + the binary tarball: + + + +alice$ cd /tmp +alice$ tar xfj nix-1.8-x86_64-darwin.tar.bz2 +alice$ cd nix-1.8-x86_64-darwin +alice$ ./install + + + + + If you need to edit the multi-user installation script to use + different group ID or a different user ID range, modify the + variables set in the file named + install-multi-user. + +
+ diff --git a/third_party/nix/doc/manual/installation/installing-source.xml b/third_party/nix/doc/manual/installation/installing-source.xml new file mode 100644 index 000000000..c261a109d --- /dev/null +++ b/third_party/nix/doc/manual/installation/installing-source.xml @@ -0,0 +1,16 @@ + + +Installing Nix from Source + +If no binary package is available, you can download and compile +a source distribution. + + + + + + diff --git a/third_party/nix/doc/manual/installation/multi-user.xml b/third_party/nix/doc/manual/installation/multi-user.xml new file mode 100644 index 000000000..69ae1ef27 --- /dev/null +++ b/third_party/nix/doc/manual/installation/multi-user.xml @@ -0,0 +1,107 @@ +
+ +Multi-User Mode + +To allow a Nix store to be shared safely among multiple users, +it is important that users are not able to run builders that modify +the Nix store or database in arbitrary ways, or that interfere with +builds started by other users. If they could do so, they could +install a Trojan horse in some package and compromise the accounts of +other users. + +To prevent this, the Nix store and database are owned by some +privileged user (usually root) and builders are +executed under special user accounts (usually named +nixbld1, nixbld2, etc.). When a +unprivileged user runs a Nix command, actions that operate on the Nix +store (such as builds) are forwarded to a Nix +daemon running under the owner of the Nix store/database +that performs the operation. + +Multi-user mode has one important limitation: only +root and a set of trusted +users specified in nix.conf can specify arbitrary +binary caches. So while unprivileged users may install packages from +arbitrary Nix expressions, they may not get pre-built +binaries. + + + + +Setting up the build users + +The build users are the special UIDs under +which builds are performed. They should all be members of the +build users group nixbld. +This group should have no other members. The build users should not +be members of any other group. On Linux, you can create the group and +users as follows: + + +$ groupadd -r nixbld +$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \ + -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \ + nixbld$n; done + + +This creates 10 build users. There can never be more concurrent builds +than the number of build users, so you may want to increase this if +you expect to do many builds at the same time. + + + + + + +Running the daemon + +The Nix daemon should be +started as follows (as root): + + +$ nix-daemon + +You’ll want to put that line somewhere in your system’s boot +scripts. + +To let unprivileged users use the daemon, they should set the +NIX_REMOTE environment +variable to daemon. So you should put a +line like + + +export NIX_REMOTE=daemon + +into the users’ login scripts. + + + + + + +Restricting access + +To limit which users can perform Nix operations, you can use the +permissions on the directory +/nix/var/nix/daemon-socket. For instance, if you +want to restrict the use of Nix to the members of a group called +nix-users, do + + +$ chgrp nix-users /nix/var/nix/daemon-socket +$ chmod ug=rwx,o= /nix/var/nix/daemon-socket + + +This way, users who are not in the nix-users group +cannot connect to the Unix domain socket +/nix/var/nix/daemon-socket/socket, so they cannot +perform Nix operations. + + + + +
diff --git a/third_party/nix/doc/manual/installation/nix-security.xml b/third_party/nix/doc/manual/installation/nix-security.xml new file mode 100644 index 000000000..d888ff14d --- /dev/null +++ b/third_party/nix/doc/manual/installation/nix-security.xml @@ -0,0 +1,27 @@ + + +Security + +Nix has two basic security models. First, it can be used in +“single-user mode”, which is similar to what most other package +management tools do: there is a single user (typically root) who performs all package +management operations. All other users can then use the installed +packages, but they cannot perform package management operations +themselves. + +Alternatively, you can configure Nix in “multi-user mode”. In +this model, all users can perform package management operations — for +instance, every user can install software without requiring root +privileges. Nix ensures that this is secure. For instance, it’s not +possible for one user to overwrite a package used by another user with +a Trojan horse. + + + + + \ No newline at end of file diff --git a/third_party/nix/doc/manual/installation/obtaining-source.xml b/third_party/nix/doc/manual/installation/obtaining-source.xml new file mode 100644 index 000000000..968822cc0 --- /dev/null +++ b/third_party/nix/doc/manual/installation/obtaining-source.xml @@ -0,0 +1,30 @@ +
+ +Obtaining a Source Distribution + +The source tarball of the most recent stable release can be +downloaded from the Nix homepage. +You can also grab the most +recent development release. + +Alternatively, the most recent sources of Nix can be obtained +from its Git +repository. For example, the following command will check out +the latest revision into a directory called +nix: + + +$ git clone https://github.com/NixOS/nix + +Likewise, specific releases can be obtained from the tags of the +repository. + +
\ No newline at end of file diff --git a/third_party/nix/doc/manual/installation/prerequisites-source.xml b/third_party/nix/doc/manual/installation/prerequisites-source.xml new file mode 100644 index 000000000..e7bdcf966 --- /dev/null +++ b/third_party/nix/doc/manual/installation/prerequisites-source.xml @@ -0,0 +1,105 @@ +
+ +Prerequisites + + + + GNU Make. + + Bash Shell. The ./configure script + relies on bashisms, so Bash is required. + + A version of GCC or Clang that supports C++17. + + pkg-config to locate + dependencies. If your distribution does not provide it, you can get + it from . + + The OpenSSL library to calculate cryptographic hashes. + If your distribution does not provide it, you can get it from . + + The libbrotlienc and + libbrotlidec libraries to provide implementation + of the Brotli compression algorithm. They are available for download + from the official repository . + + The bzip2 compressor program and the + libbz2 library. Thus you must have bzip2 + installed, including development headers and libraries. If your + distribution does not provide these, you can obtain bzip2 from . + + liblzma, which is provided by + XZ Utils. If your distribution does not provide this, you can + get it from . + + cURL and its library. If your distribution does not + provide it, you can get it from . + + The SQLite embedded database library, version 3.6.19 + or higher. If your distribution does not provide it, please install + it from . + + The Boehm + garbage collector to reduce the evaluator’s memory + consumption (optional). To enable it, install + pkgconfig and the Boehm garbage collector, and + pass the flag to + configure. + + The boost library of version + 1.66.0 or higher. It can be obtained from the official web site + . + + The editline library of version + 1.14.0 or higher. It can be obtained from the its repository + . + + The xmllint and + xsltproc programs to build this manual and the + man-pages. These are part of the libxml2 and + libxslt packages, respectively. You also need + the DocBook + XSL stylesheets and optionally the DocBook 5.0 RELAX NG + schemas. Note that these are only required if you modify the + manual sources or when you are building from the Git + repository. + + Recent versions of Bison and Flex to build the + parser. (This is because Nix needs GLR support in Bison and + reentrancy support in Flex.) For Bison, you need version 2.6, which + can be obtained from the GNU FTP + server. For Flex, you need version 2.5.35, which is + available on SourceForge. + Slightly older versions may also work, but ancient versions like the + ubiquitous 2.5.4a won't. Note that these are only required if you + modify the parser or when you are building from the Git + repository. + + The libseccomp is used to provide + syscall filtering on Linux. This is an optional dependency and can + be disabled passing a + option to the configure script (Not recommended + unless your system doesn't support + libseccomp). To get the library, visit . + + + +
diff --git a/third_party/nix/doc/manual/installation/single-user.xml b/third_party/nix/doc/manual/installation/single-user.xml new file mode 100644 index 000000000..09cdaa5d4 --- /dev/null +++ b/third_party/nix/doc/manual/installation/single-user.xml @@ -0,0 +1,21 @@ +
+ +Single-User Mode + +In single-user mode, all Nix operations that access the database +in prefix/var/nix/db +or modify the Nix store in +prefix/store must be +performed under the user ID that owns those directories. This is +typically root. (If you +install from RPM packages, that’s in fact the default ownership.) +However, on single-user machines, it is often convenient to +chown those directories to your normal user account +so that you don’t have to su to root all the time. + +
\ No newline at end of file diff --git a/third_party/nix/doc/manual/installation/supported-platforms.xml b/third_party/nix/doc/manual/installation/supported-platforms.xml new file mode 100644 index 000000000..3e74be49d --- /dev/null +++ b/third_party/nix/doc/manual/installation/supported-platforms.xml @@ -0,0 +1,36 @@ + + +Supported Platforms + +Nix is currently supported on the following platforms: + + + + Linux (i686, x86_64, aarch64). + + macOS (x86_64). + + + + + + + + + + diff --git a/third_party/nix/doc/manual/installation/upgrading.xml b/third_party/nix/doc/manual/installation/upgrading.xml new file mode 100644 index 000000000..30670d7fe --- /dev/null +++ b/third_party/nix/doc/manual/installation/upgrading.xml @@ -0,0 +1,22 @@ + + + Upgrading Nix + + + Multi-user Nix users on macOS can upgrade Nix by running: + sudo -i sh -c 'nix-channel --update && + nix-env -iA nixpkgs.nix && + launchctl remove org.nixos.nix-daemon && + launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist' + + + + + Single-user installations of Nix should run this: + nix-channel --update; nix-env -iA nixpkgs.nix + + diff --git a/third_party/nix/doc/manual/introduction/about-nix.xml b/third_party/nix/doc/manual/introduction/about-nix.xml new file mode 100644 index 000000000..c21ed34dd --- /dev/null +++ b/third_party/nix/doc/manual/introduction/about-nix.xml @@ -0,0 +1,268 @@ + + +About Nix + +Nix is a purely functional package manager. +This means that it treats packages like values in purely functional +programming languages such as Haskell — they are built by functions +that don’t have side-effects, and they never change after they have +been built. Nix stores packages in the Nix +store, usually the directory +/nix/store, where each package has its own unique +subdirectory such as + + +/nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1/ + + +where b6gvzjyb2pg0… is a unique identifier for the +package that captures all its dependencies (it’s a cryptographic hash +of the package’s build dependency graph). This enables many powerful +features. + + +Multiple versions + +You can have multiple versions or variants of a package +installed at the same time. This is especially important when +different applications have dependencies on different versions of the +same package — it prevents the “DLL hell”. Because of the hashing +scheme, different versions of a package end up in different paths in +the Nix store, so they don’t interfere with each other. + +An important consequence is that operations like upgrading or +uninstalling an application cannot break other applications, since +these operations never “destructively” update or delete files that are +used by other packages. + + + + +Complete dependencies + +Nix helps you make sure that package dependency specifications +are complete. In general, when you’re making a package for a package +management system like RPM, you have to specify for each package what +its dependencies are, but there are no guarantees that this +specification is complete. If you forget a dependency, then the +package will build and work correctly on your +machine if you have the dependency installed, but not on the end +user's machine if it's not there. + +Since Nix on the other hand doesn’t install packages in “global” +locations like /usr/bin but in package-specific +directories, the risk of incomplete dependencies is greatly reduced. +This is because tools such as compilers don’t search in per-packages +directories such as +/nix/store/5lbfaxb722zp…-openssl-0.9.8d/include, +so if a package builds correctly on your system, this is because you +specified the dependency explicitly. This takes care of the build-time +dependencies. + +Once a package is built, runtime dependencies are found by +scanning binaries for the hash parts of Nix store paths (such as +r8vvq9kq…). This sounds risky, but it works +extremely well. + + + + +Multi-user support + +Nix has multi-user support. This means that non-privileged +users can securely install software. Each user can have a different +profile, a set of packages in the Nix store that +appear in the user’s PATH. If a user installs a +package that another user has already installed previously, the +package won’t be built or downloaded a second time. At the same time, +it is not possible for one user to inject a Trojan horse into a +package that might be used by another user. + + + + +Atomic upgrades and rollbacks + +Since package management operations never overwrite packages in +the Nix store but just add new versions in different paths, they are +atomic. So during a package upgrade, there is no +time window in which the package has some files from the old version +and some files from the new version — which would be bad because a +program might well crash if it’s started during that period. + +And since packages aren’t overwritten, the old versions are still +there after an upgrade. This means that you can roll +back to the old version: + + +$ nix-env --upgrade some-packages +$ nix-env --rollback + + + + + +Garbage collection + +When you uninstall a package like this… + + +$ nix-env --uninstall firefox + + +the package isn’t deleted from the system right away (after all, you +might want to do a rollback, or it might be in the profiles of other +users). Instead, unused packages can be deleted safely by running the +garbage collector: + + +$ nix-collect-garbage + + +This deletes all packages that aren’t in use by any user profile or by +a currently running program. + + + + +Functional package language + +Packages are built from Nix expressions, +which is a simple functional language. A Nix expression describes +everything that goes into a package build action (a “derivation”): +other packages, sources, the build script, environment variables for +the build script, etc. Nix tries very hard to ensure that Nix +expressions are deterministic: building a Nix +expression twice should yield the same result. + +Because it’s a functional language, it’s easy to support +building variants of a package: turn the Nix expression into a +function and call it any number of times with the appropriate +arguments. Due to the hashing scheme, variants don’t conflict with +each other in the Nix store. + + + + +Transparent source/binary deployment + +Nix expressions generally describe how to build a package from +source, so an installation action like + + +$ nix-env --install firefox + + +could cause quite a bit of build activity, as not +only Firefox but also all its dependencies (all the way up to the C +library and the compiler) would have to built, at least if they are +not already in the Nix store. This is a source deployment +model. For most users, building from source is not very +pleasant as it takes far too long. However, Nix can automatically +skip building from source and instead use a binary +cache, a web server that provides pre-built binaries. For +instance, when asked to build +/nix/store/b6gvzjyb2pg0…-firefox-33.1 from source, +Nix would first check if the file +https://cache.nixos.org/b6gvzjyb2pg0….narinfo exists, and +if so, fetch the pre-built binary referenced from there; otherwise, it +would fall back to building from source. + + + + + + + +Nix Packages collection + +We provide a large set of Nix expressions containing hundreds of +existing Unix packages, the Nix Packages +collection (Nixpkgs). + + + + +Managing build environments + +Nix is extremely useful for developers as it makes it easy to +automatically set up the build environment for a package. Given a +Nix expression that describes the dependencies of your package, the +command nix-shell will build or download those +dependencies if they’re not already in your Nix store, and then start +a Bash shell in which all necessary environment variables (such as +compiler search paths) are set. + +For example, the following command gets all dependencies of the +Pan newsreader, as described by its +Nix expression: + + +$ nix-shell '<nixpkgs>' -A pan + + +You’re then dropped into a shell where you can edit, build and test +the package: + + +[nix-shell]$ tar xf $src +[nix-shell]$ cd pan-* +[nix-shell]$ ./configure +[nix-shell]$ make +[nix-shell]$ ./pan/gui/pan + + + + + + + +Portability + +Nix runs on Linux and macOS. + + + + +NixOS + +NixOS is a Linux distribution based on Nix. It uses Nix not +just for package management but also to manage the system +configuration (e.g., to build configuration files in +/etc). This means, among other things, that it +is easy to roll back the entire configuration of the system to an +earlier state. Also, users can install software without root +privileges. For more information and downloads, see the NixOS homepage. + + + + +License + +Nix is released under the terms of the GNU +LGPLv2.1 or (at your option) any later version. + + + + + diff --git a/third_party/nix/doc/manual/introduction/introduction.xml b/third_party/nix/doc/manual/introduction/introduction.xml new file mode 100644 index 000000000..12b2cc761 --- /dev/null +++ b/third_party/nix/doc/manual/introduction/introduction.xml @@ -0,0 +1,12 @@ + + +Introduction + + + + + diff --git a/third_party/nix/doc/manual/introduction/quick-start.xml b/third_party/nix/doc/manual/introduction/quick-start.xml new file mode 100644 index 000000000..1ce6c8d50 --- /dev/null +++ b/third_party/nix/doc/manual/introduction/quick-start.xml @@ -0,0 +1,124 @@ + + +Quick Start + +This chapter is for impatient people who don't like reading +documentation. For more in-depth information you are kindly referred +to subsequent chapters. + + + +Install single-user Nix by running the following: + + +$ bash <(curl https://nixos.org/nix/install) + + +This will install Nix in /nix. The install script +will create /nix using sudo, +so make sure you have sufficient rights. (For other installation +methods, see .) + +See what installable packages are currently available +in the channel: + + +$ nix-env -qa +docbook-xml-4.3 +docbook-xml-4.5 +firefox-33.0.2 +hello-2.9 +libxslt-1.1.28 +... + + + +Install some packages from the channel: + + +$ nix-env -i hello + +This should download pre-built packages; it should not build them +locally (if it does, something went wrong). + +Test that they work: + + +$ which hello +/home/eelco/.nix-profile/bin/hello +$ hello +Hello, world! + + + + +Uninstall a package: + + +$ nix-env -e hello + + + +You can also test a package without installing it: + + +$ nix-shell -p hello + + +This builds or downloads GNU Hello and its dependencies, then drops +you into a Bash shell where the hello command is +present, all without affecting your normal environment: + + +[nix-shell:~]$ hello +Hello, world! + +[nix-shell:~]$ exit + +$ hello +hello: command not found + + + + +To keep up-to-date with the channel, do: + + +$ nix-channel --update nixpkgs +$ nix-env -u '*' + +The latter command will upgrade each installed package for which there +is a “newer” version (as determined by comparing the version +numbers). + +If you're unhappy with the result of a +nix-env action (e.g., an upgraded package turned +out not to work properly), you can go back: + + +$ nix-env --rollback + + + +You should periodically run the Nix garbage collector +to get rid of unused packages, since uninstalls or upgrades don't +actually delete them: + + +$ nix-collect-garbage -d + + + + + + + + diff --git a/third_party/nix/doc/manual/local.mk b/third_party/nix/doc/manual/local.mk new file mode 100644 index 000000000..b4e7f35d5 --- /dev/null +++ b/third_party/nix/doc/manual/local.mk @@ -0,0 +1,86 @@ + +ifeq ($(doc_generate),yes) + +XSLTPROC = $(xsltproc) --nonet $(xmlflags) \ + --param section.autolabel 1 \ + --param section.label.includes.component.label 1 \ + --param xref.with.number.and.title 1 \ + --param toc.section.depth 3 \ + --param admon.style \'\' \ + --param callout.graphics.extension \'.gif\' \ + --param contrib.inline.enabled 0 \ + --stringparam generate.toc "book toc" \ + --param keep.relative.image.uris 0 + +docbookxsl = http://docbook.sourceforge.net/release/xsl-ns/current +docbookrng = http://docbook.org/xml/5.0/rng/docbook.rng + +MANUAL_SRCS := $(call rwildcard, $(d), *.xml) + + +# Do XInclude processing / RelaxNG validation +$(d)/manual.xmli: $(d)/manual.xml $(MANUAL_SRCS) $(d)/version.txt + $(trace-gen) $(xmllint) --nonet --xinclude $< -o $@.tmp + @mv $@.tmp $@ + +$(d)/version.txt: + $(trace-gen) echo -n $(PACKAGE_VERSION) > $@ + +# Note: RelaxNG validation requires xmllint >= 2.7.4. +$(d)/manual.is-valid: $(d)/manual.xmli + $(trace-gen) $(XSLTPROC) --novalid --stringparam profile.condition manual \ + $(docbookxsl)/profiling/profile.xsl $< 2> /dev/null | \ + $(xmllint) --nonet --noout --relaxng $(docbookrng) - + @touch $@ + +clean-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid + +dist-files += $(d)/manual.xmli $(d)/version.txt $(d)/manual.is-valid + + +# Generate man pages. +man-pages := $(foreach n, \ + nix-env.1 nix-build.1 nix-shell.1 nix-store.1 nix-instantiate.1 \ + nix-collect-garbage.1 \ + nix-prefetch-url.1 nix-channel.1 \ + nix-hash.1 nix-copy-closure.1 \ + nix.conf.5 nix-daemon.8, \ + $(d)/$(n)) + +$(firstword $(man-pages)): $(d)/manual.xmli $(d)/manual.is-valid + $(trace-gen) $(XSLTPROC) --novalid --stringparam profile.condition manpage \ + $(docbookxsl)/profiling/profile.xsl $< 2> /dev/null | \ + (cd doc/manual && $(XSLTPROC) $(docbookxsl)/manpages/docbook.xsl -) + +$(wordlist 2, $(words $(man-pages)), $(man-pages)): $(firstword $(man-pages)) + +clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 + +dist-files += $(man-pages) + + +# Generate the HTML manual. +$(d)/manual.html: $(d)/manual.xml $(MANUAL_SRCS) $(d)/manual.is-valid + $(trace-gen) $(XSLTPROC) --xinclude --stringparam profile.condition manual \ + $(docbookxsl)/profiling/profile.xsl $< | \ + $(XSLTPROC) --output $@ $(docbookxsl)/xhtml/docbook.xsl - + +$(foreach file, $(d)/manual.html, $(eval $(call install-data-in, $(file), $(docdir)/manual))) + +$(foreach file, $(wildcard $(d)/figures/*.png), $(eval $(call install-data-in, $(file), $(docdir)/manual/figures))) + +$(foreach file, $(wildcard $(d)/images/callouts/*.gif), $(eval $(call install-data-in, $(file), $(docdir)/manual/images/callouts))) + +$(eval $(call install-symlink, manual.html, $(docdir)/manual/index.html)) + + +all: $(d)/manual.html + + + +clean-files += $(d)/manual.html + +dist-files += $(d)/manual.html + + +endif diff --git a/third_party/nix/doc/manual/manual.xml b/third_party/nix/doc/manual/manual.xml new file mode 100644 index 000000000..87d9de28a --- /dev/null +++ b/third_party/nix/doc/manual/manual.xml @@ -0,0 +1,52 @@ + + + + Nix Package Manager Guide + Version + + + + Eelco + Dolstra + + Author + + + + 2004-2018 + Eelco Dolstra + + + + + + + + + + + + + + + + + + + + diff --git a/third_party/nix/doc/manual/nix-lang-ref.xml b/third_party/nix/doc/manual/nix-lang-ref.xml new file mode 100644 index 000000000..86273ac3d --- /dev/null +++ b/third_party/nix/doc/manual/nix-lang-ref.xml @@ -0,0 +1,182 @@ + + Nix Language Reference + + + Grammar + + + Expressions + + + Expr + + + + + + + ExprFunction + + '{' '}' ':' + | + + + + + + ExprAssert + + 'assert' ';' + | + + + + + + ExprIf + + 'if' 'then' + 'else' + | + + + + + + ExprOp + + '!' + | + '==' + | + '!=' + | + '&&' + | + '||' + | + '->' + | + '//' + | + '~' + | + '?' + | + + + + + + ExprApp + + '.' + | + + + + + + ExprSelect + + + | + + + + + + ExprSimple + + | + | + | + | + + | + 'true' | 'false' | 'null' + | + '(' ')' + | + '{' * '}' + | + 'let' '{' * '}' + | + 'rec' '{' * '}' + | + '[' * ']' + + + + + Bind + + '=' ';' + | + 'inherit' ('(' ')')? * ';' + + + + + Formals + + ',' + | + + + + + Formal + + + | + '?' + + + + + + + Terminals + + + Id + [a-zA-Z\_][a-zA-Z0-9\_\']* + + + + Int + [0-9]+ + + + + Str + \"[^\n\"]*\" + + + + Path + [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+ + + + + Uri + [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ + + + + Whitespace + + [ \t\n]+ + | + \#[^\n]* + | + \/\*(.|\n)*\*\/ + + + + + + + + diff --git a/third_party/nix/doc/manual/packages/basic-package-mgmt.xml b/third_party/nix/doc/manual/packages/basic-package-mgmt.xml new file mode 100644 index 000000000..0f21297f3 --- /dev/null +++ b/third_party/nix/doc/manual/packages/basic-package-mgmt.xml @@ -0,0 +1,194 @@ + + +Basic Package Management + +The main command for package management is nix-env. You can use +it to install, upgrade, and erase packages, and to query what +packages are installed or are available for installation. + +In Nix, different users can have different “views” +on the set of installed applications. That is, there might be lots of +applications present on the system (possibly in many different +versions), but users can have a specific selection of those active — +where “active” just means that it appears in a directory +in the user’s PATH. Such a view on the set of +installed applications is called a user +environment, which is just a directory tree consisting of +symlinks to the files of the active applications. + +Components are installed from a set of Nix +expressions that tell Nix how to build those packages, +including, if necessary, their dependencies. There is a collection of +Nix expressions called the Nixpkgs package collection that contains +packages ranging from basic development stuff such as GCC and Glibc, +to end-user applications like Mozilla Firefox. (Nix is however not +tied to the Nixpkgs package collection; you could write your own Nix +expressions based on Nixpkgs, or completely new ones.) + +You can manually download the latest version of Nixpkgs from +. However, +it’s much more convenient to use the Nixpkgs +channel, since it makes it easy to stay up to +date with new versions of Nixpkgs. (Channels are described in more +detail in .) Nixpkgs is automatically +added to your list of “subscribed” channels when you install +Nix. If this is not the case for some reason, you can add it as +follows: + + +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable +$ nix-channel --update + + + + +On NixOS, you’re automatically subscribed to a NixOS +channel corresponding to your NixOS major release +(e.g. http://nixos.org/channels/nixos-14.12). A NixOS +channel is identical to the Nixpkgs channel, except that it contains +only Linux binaries and is updated only if a set of regression tests +succeed. + +You can view the set of available packages in Nixpkgs: + + +$ nix-env -qa +aterm-2.2 +bash-3.0 +binutils-2.15 +bison-1.875d +blackdown-1.4.2 +bzip2-1.0.2 +… + +The flag specifies a query operation, and + means that you want to show the “available” (i.e., +installable) packages, as opposed to the installed packages. If you +downloaded Nixpkgs yourself, or if you checked it out from GitHub, +then you need to pass the path to your Nixpkgs tree using the + flag: + + +$ nix-env -qaf /path/to/nixpkgs + + +where /path/to/nixpkgs is where you’ve +unpacked or checked out Nixpkgs. + +You can select specific packages by name: + + +$ nix-env -qa firefox +firefox-34.0.5 +firefox-with-plugins-34.0.5 + + +and using regular expressions: + + +$ nix-env -qa 'firefox.*' + + + + +It is also possible to see the status of +available packages, i.e., whether they are installed into the user +environment and/or present in the system: + + +$ nix-env -qas +… +-PS bash-3.0 +--S binutils-2.15 +IPS bison-1.875d +… + +The first character (I) indicates whether the +package is installed in your current user environment. The second +(P) indicates whether it is present on your system +(in which case installing it into your user environment would be a +very quick operation). The last one (S) indicates +whether there is a so-called substitute for the +package, which is Nix’s mechanism for doing binary deployment. It +just means that Nix knows that it can fetch a pre-built package from +somewhere (typically a network server) instead of building it +locally. + +You can install a package using nix-env -i. +For instance, + + +$ nix-env -i subversion + +will install the package called subversion (which +is, of course, the Subversion version +management system). + +When you ask Nix to install a package, it will first try +to get it in pre-compiled form from a binary +cache. By default, Nix will use the binary cache +https://cache.nixos.org; it contains binaries for most +packages in Nixpkgs. Only if no binary is available in the binary +cache, Nix will build the package from source. So if nix-env +-i subversion results in Nix building stuff from source, +then either the package is not built for your platform by the Nixpkgs +build servers, or your version of Nixpkgs is too old or too new. For +instance, if you have a very recent checkout of Nixpkgs, then the +Nixpkgs build servers may not have had a chance to build everything +and upload the resulting binaries to +https://cache.nixos.org. The Nixpkgs channel is only +updated after all binaries have been uploaded to the cache, so if you +stick to the Nixpkgs channel (rather than using a Git checkout of the +Nixpkgs tree), you will get binaries for most packages. + +Naturally, packages can also be uninstalled: + + +$ nix-env -e subversion + + + +Upgrading to a new version is just as easy. If you have a new +release of Nix Packages, you can do: + + +$ nix-env -u subversion + +This will only upgrade Subversion if there is a +“newer” version in the new set of Nix expressions, as +defined by some pretty arbitrary rules regarding ordering of version +numbers (which generally do what you’d expect of them). To just +unconditionally replace Subversion with whatever version is in the Nix +expressions, use -i instead of +-u; -i will remove +whatever version is already installed. + +You can also upgrade all packages for which there are newer +versions: + + +$ nix-env -u + + + +Sometimes it’s useful to be able to ask what +nix-env would do, without actually doing it. For +instance, to find out what packages would be upgraded by +nix-env -u, you can do + + +$ nix-env -u --dry-run +(dry run; not doing anything) +upgrading `libxslt-1.1.0' to `libxslt-1.1.10' +upgrading `graphviz-1.10' to `graphviz-1.12' +upgrading `coreutils-5.0' to `coreutils-5.2.1' + + + + diff --git a/third_party/nix/doc/manual/packages/binary-cache-substituter.xml b/third_party/nix/doc/manual/packages/binary-cache-substituter.xml new file mode 100644 index 000000000..c6ceb9c80 --- /dev/null +++ b/third_party/nix/doc/manual/packages/binary-cache-substituter.xml @@ -0,0 +1,70 @@ +
+ +Serving a Nix store via HTTP + +You can easily share the Nix store of a machine via HTTP. This +allows other machines to fetch store paths from that machine to speed +up installations. It uses the same binary cache +mechanism that Nix usually uses to fetch pre-built binaries from +https://cache.nixos.org. + +The daemon that handles binary cache requests via HTTP, +nix-serve, is not part of the Nix distribution, but +you can install it from Nixpkgs: + + +$ nix-env -i nix-serve + + +You can then start the server, listening for HTTP connections on +whatever port you like: + + +$ nix-serve -p 8080 + + +To check whether it works, try the following on the client: + + +$ curl http://avalon:8080/nix-cache-info + + +which should print something like: + + +StoreDir: /nix/store +WantMassQuery: 1 +Priority: 30 + + + + +On the client side, you can tell Nix to use your binary cache +using , e.g.: + + +$ nix-env -i firefox --option extra-binary-caches http://avalon:8080/ + + +The option tells Nix to use this +binary cache in addition to your default caches, such as +https://cache.nixos.org. Thus, for any path in the closure +of Firefox, Nix will first check if the path is available on the +server avalon or another binary caches. If not, it +will fall back to building from source. + +You can also tell Nix to always use your binary cache by adding +a line to the nix.conf +configuration file like this: + + +binary-caches = http://avalon:8080/ https://cache.nixos.org/ + + + + +
diff --git a/third_party/nix/doc/manual/packages/channels.xml b/third_party/nix/doc/manual/packages/channels.xml new file mode 100644 index 000000000..15c119fcb --- /dev/null +++ b/third_party/nix/doc/manual/packages/channels.xml @@ -0,0 +1,57 @@ + + +Channels + +If you want to stay up to date with a set of packages, it’s not +very convenient to manually download the latest set of Nix expressions +for those packages and upgrade using nix-env. +Fortunately, there’s a better way: Nix +channels. + +A Nix channel is just a URL that points to a place that contains +a set of Nix expressions and a manifest. Using the command nix-channel you +can automatically stay up to date with whatever is available at that +URL. + +You can “subscribe” to a channel using +nix-channel --add, e.g., + + +$ nix-channel --add https://nixos.org/channels/nixpkgs-unstable + +subscribes you to a channel that always contains that latest version +of the Nix Packages collection. (Subscribing really just means that +the URL is added to the file ~/.nix-channels, +where it is read by subsequent calls to nix-channel +--update.) You can “unsubscribe” using nix-channel +--remove: + + +$ nix-channel --remove nixpkgs + + + +To obtain the latest Nix expressions available in a channel, do + + +$ nix-channel --update + +This downloads and unpacks the Nix expressions in every channel +(downloaded from url/nixexprs.tar.bz2). +It also makes the union of each channel’s Nix expressions available by +default to nix-env operations (via the symlink +~/.nix-defexpr/channels). Consequently, you can +then say + + +$ nix-env -u + +to upgrade all packages in your profile to the latest versions +available in the subscribed channels. + + diff --git a/third_party/nix/doc/manual/packages/copy-closure.xml b/third_party/nix/doc/manual/packages/copy-closure.xml new file mode 100644 index 000000000..012030e3e --- /dev/null +++ b/third_party/nix/doc/manual/packages/copy-closure.xml @@ -0,0 +1,50 @@ +
+ +Copying Closures Via SSH + +The command nix-copy-closure copies a Nix +store path along with all its dependencies to or from another machine +via the SSH protocol. It doesn’t copy store paths that are already +present on the target machine. For example, the following command +copies Firefox with all its dependencies: + + +$ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) + +See for details. + +With nix-store +--export and nix-store --import you can +write the closure of a store path (that is, the path and all its +dependencies) to a file, and then unpack that file into another Nix +store. For example, + + +$ nix-store --export $(nix-store -qR $(type -p firefox)) > firefox.closure + +writes the closure of Firefox to a file. You can then copy this file +to another machine and install the closure: + + +$ nix-store --import < firefox.closure + +Any store paths in the closure that are already present in the target +store are ignored. It is also possible to pipe the export into +another command, e.g. to copy and install a closure directly to/on +another machine: + + +$ nix-store --export $(nix-store -qR $(type -p firefox)) | bzip2 | \ + ssh alice@itchy.example.org "bunzip2 | nix-store --import" + +However, nix-copy-closure is generally more +efficient because it only copies paths that are not already present in +the target Nix store. + +
diff --git a/third_party/nix/doc/manual/packages/garbage-collection.xml b/third_party/nix/doc/manual/packages/garbage-collection.xml new file mode 100644 index 000000000..b506f22b0 --- /dev/null +++ b/third_party/nix/doc/manual/packages/garbage-collection.xml @@ -0,0 +1,86 @@ + + +Garbage Collection + +nix-env operations such as upgrades +() and uninstall () never +actually delete packages from the system. All they do (as shown +above) is to create a new user environment that no longer contains +symlinks to the “deleted” packages. + +Of course, since disk space is not infinite, unused packages +should be removed at some point. You can do this by running the Nix +garbage collector. It will remove from the Nix store any package +not used (directly or indirectly) by any generation of any +profile. + +Note however that as long as old generations reference a +package, it will not be deleted. After all, we wouldn’t be able to +do a rollback otherwise. So in order for garbage collection to be +effective, you should also delete (some) old generations. Of course, +this should only be done if you are certain that you will not need to +roll back. + +To delete all old (non-current) generations of your current +profile: + + +$ nix-env --delete-generations old + +Instead of old you can also specify a list of +generations, e.g., + + +$ nix-env --delete-generations 10 11 14 + +To delete all generations older than a specified number of days +(except the current generation), use the d +suffix. For example, + + +$ nix-env --delete-generations 14d + +deletes all generations older than two weeks. + +After removing appropriate old generations you can run the +garbage collector as follows: + + +$ nix-store --gc + +The behaviour of the gargage collector is affected by the +keep-derivations (default: true) and keep-outputs +(default: false) options in the Nix configuration file. The defaults will ensure +that all derivations that are build-time dependencies of garbage collector roots +will be kept and that all output paths that are runtime dependencies +will be kept as well. All other derivations or paths will be collected. +(This is usually what you want, but while you are developing +it may make sense to keep outputs to ensure that rebuild times are quick.) + +If you are feeling uncertain, you can also first view what files would +be deleted: + + +$ nix-store --gc --print-dead + +Likewise, the option will show the paths +that won’t be deleted. + +There is also a convenient little utility +nix-collect-garbage, which when invoked with the + () switch deletes all +old generations of all profiles in +/nix/var/nix/profiles. So + + +$ nix-collect-garbage -d + +is a quick and easy way to clean up your system. + + + + diff --git a/third_party/nix/doc/manual/packages/garbage-collector-roots.xml b/third_party/nix/doc/manual/packages/garbage-collector-roots.xml new file mode 100644 index 000000000..8338e5392 --- /dev/null +++ b/third_party/nix/doc/manual/packages/garbage-collector-roots.xml @@ -0,0 +1,29 @@ +
+ +Garbage Collector Roots + +The roots of the garbage collector are all store paths to which +there are symlinks in the directory +prefix/nix/var/nix/gcroots. +For instance, the following command makes the path +/nix/store/d718ef...-foo a root of the collector: + + +$ ln -s /nix/store/d718ef...-foo /nix/var/nix/gcroots/bar + +That is, after this command, the garbage collector will not remove +/nix/store/d718ef...-foo or any of its +dependencies. + +Subdirectories of +prefix/nix/var/nix/gcroots +are also searched for symlinks. Symlinks to non-store paths are +followed and searched for roots, but symlinks to non-store paths +inside the paths reached in that way are not +followed to prevent infinite recursion. + +
\ No newline at end of file diff --git a/third_party/nix/doc/manual/packages/package-management.xml b/third_party/nix/doc/manual/packages/package-management.xml new file mode 100644 index 000000000..61e55faeb --- /dev/null +++ b/third_party/nix/doc/manual/packages/package-management.xml @@ -0,0 +1,23 @@ + + +Package Management + + +This chapter discusses how to do package management with Nix, +i.e., how to obtain, install, upgrade, and erase packages. This is +the “user’s” perspective of the Nix system — people +who want to create packages should consult +. + + + + + + + + + diff --git a/third_party/nix/doc/manual/packages/profiles.xml b/third_party/nix/doc/manual/packages/profiles.xml new file mode 100644 index 000000000..4d10319ab --- /dev/null +++ b/third_party/nix/doc/manual/packages/profiles.xml @@ -0,0 +1,158 @@ + + +Profiles + +Profiles and user environments are Nix’s mechanism for +implementing the ability to allow different users to have different +configurations, and to do atomic upgrades and rollbacks. To +understand how they work, it’s useful to know a bit about how Nix +works. In Nix, packages are stored in unique locations in the +Nix store (typically, +/nix/store). For instance, a particular version +of the Subversion package might be stored in a directory +/nix/store/dpmvp969yhdqs7lm2r1a3gng7pyq6vy4-subversion-1.1.3/, +while another version might be stored in +/nix/store/5mq2jcn36ldlmh93yj1n8s9c95pj7c5s-subversion-1.1.2. +The long strings prefixed to the directory names are cryptographic +hashes160-bit truncations of SHA-256 hashes encoded in +a base-32 notation, to be precise. of +all inputs involved in building the package — +sources, dependencies, compiler flags, and so on. So if two +packages differ in any way, they end up in different locations in +the file system, so they don’t interfere with each other. shows a part of a typical Nix +store. + +
User environments + + + + + +
+ +Of course, you wouldn’t want to type + + +$ /nix/store/dpmvp969yhdq...-subversion-1.1.3/bin/svn + +every time you want to run Subversion. Of course we could set up the +PATH environment variable to include the +bin directory of every package we want to use, +but this is not very convenient since changing PATH +doesn’t take effect for already existing processes. The solution Nix +uses is to create directory trees of symlinks to +activated packages. These are called +user environments and they are packages +themselves (though automatically generated by +nix-env), so they too reside in the Nix store. For +instance, in the user +environment /nix/store/0c1p5z4kda11...-user-env +contains a symlink to just Subversion 1.1.2 (arrows in the figure +indicate symlinks). This would be what we would obtain if we had done + + +$ nix-env -i subversion + +on a set of Nix expressions that contained Subversion 1.1.2. + +This doesn’t in itself solve the problem, of course; you +wouldn’t want to type +/nix/store/0c1p5z4kda11...-user-env/bin/svn +either. That’s why there are symlinks outside of the store that point +to the user environments in the store; for instance, the symlinks +default-42-link and +default-43-link in the example. These are called +generations since every time you perform a +nix-env operation, a new user environment is +generated based on the current one. For instance, generation 43 was +created from generation 42 when we did + + +$ nix-env -i subversion firefox + +on a set of Nix expressions that contained Firefox and a new version +of Subversion. + +Generations are grouped together into +profiles so that different users don’t interfere +with each other if they don’t want to. For example: + + +$ ls -l /nix/var/nix/profiles/ +... +lrwxrwxrwx 1 eelco ... default-42-link -> /nix/store/0c1p5z4kda11...-user-env +lrwxrwxrwx 1 eelco ... default-43-link -> /nix/store/3aw2pdyx2jfc...-user-env +lrwxrwxrwx 1 eelco ... default -> default-43-link + +This shows a profile called default. The file +default itself is actually a symlink that points +to the current generation. When we do a nix-env +operation, a new user environment and generation link are created +based on the current one, and finally the default +symlink is made to point at the new generation. This last step is +atomic on Unix, which explains how we can do atomic upgrades. (Note +that the building/installing of new packages doesn’t interfere in +any way with old packages, since they are stored in different +locations in the Nix store.) + +If you find that you want to undo a nix-env +operation, you can just do + + +$ nix-env --rollback + +which will just make the current generation link point at the previous +link. E.g., default would be made to point at +default-42-link. You can also switch to a +specific generation: + + +$ nix-env --switch-generation 43 + +which in this example would roll forward to generation 43 again. You +can also see all available generations: + + +$ nix-env --list-generations + +You generally wouldn’t have +/nix/var/nix/profiles/some-profile/bin +in your PATH. Rather, there is a symlink +~/.nix-profile that points to your current +profile. This means that you should put +~/.nix-profile/bin in your PATH +(and indeed, that’s what the initialisation script +/nix/etc/profile.d/nix.sh does). This makes it +easier to switch to a different profile. You can do that using the +command nix-env --switch-profile: + + +$ nix-env --switch-profile /nix/var/nix/profiles/my-profile + +$ nix-env --switch-profile /nix/var/nix/profiles/default + +These commands switch to the my-profile and +default profile, respectively. If the profile doesn’t exist, it will +be created automatically. You should be careful about storing a +profile in another location than the profiles +directory, since otherwise it might not be used as a root of the +garbage collector (see ). + +All nix-env operations work on the profile +pointed to by ~/.nix-profile, but you can override +this using the option (abbreviation +): + + +$ nix-env -p /nix/var/nix/profiles/other-profile -i subversion + +This will not change the +~/.nix-profile symlink. + + diff --git a/third_party/nix/doc/manual/packages/s3-substituter.xml b/third_party/nix/doc/manual/packages/s3-substituter.xml new file mode 100644 index 000000000..868b5a66d --- /dev/null +++ b/third_party/nix/doc/manual/packages/s3-substituter.xml @@ -0,0 +1,182 @@ + +
+ +Serving a Nix store via AWS S3 or S3-compatible Service + +Nix has built-in support for storing and fetching store paths +from Amazon S3 and S3 compatible services. This uses the same +binary cache mechanism that Nix usually uses to +fetch prebuilt binaries from cache.nixos.org. + +The following options can be specified as URL parameters to +the S3 URL: + + + profile + + + The name of the AWS configuration profile to use. By default + Nix will use the default profile. + + + + + region + + + The region of the S3 bucket. us–east-1 by + default. + + + + If your bucket is not in us–east-1, you + should always explicitly specify the region parameter. + + + + + endpoint + + + The URL to your S3-compatible service, for when not using + Amazon S3. Do not specify this value if you're using Amazon + S3. + + This endpoint must support HTTPS and will use + path-based addressing instead of virtual host based + addressing. + + + + scheme + + + The scheme used for S3 requests, https + (default) or http. This option allows you to + disable HTTPS for binary caches which don't support it. + + HTTPS should be used if the cache might contain + sensitive information. + + + + +In this example we will use the bucket named +example-nix-cache. + +
+ Anonymous Reads to your S3-compatible binary cache + + If your binary cache is publicly accessible and does not + require authentication, the simplest and easiest way to use Nix with + your S3 compatible binary cache is to use the HTTP URL for that + cache. + + For AWS S3 the binary cache URL for example bucket will be + exactly https://example-nix-cache.s3.amazonaws.com or + s3://example-nix-cache. For S3 compatible binary caches, + consult that cache's documentation. + + Your bucket will need the following bucket policy: + + +
+ +
+ Authenticated Reads to your S3 binary cache + + For AWS S3 the binary cache URL for example bucket will be + exactly s3://example-nix-cache. + + Nix will use the default + credential provider chain for authenticating requests to + Amazon S3. + + Nix supports authenticated reads from Amazon S3 and S3 + compatible binary caches. + + Your bucket will need a bucket policy allowing the desired + users to perform the s3:GetObject and + s3:GetBucketLocation action on all objects in the + bucket. The anonymous policy in can be updated to + have a restricted Principal to support + this. +
+ + +
+ Authenticated Writes to your S3-compatible binary cache + + Nix support fully supports writing to Amazon S3 and S3 + compatible buckets. The binary cache URL for our example bucket will + be s3://example-nix-cache. + + Nix will use the default + credential provider chain for authenticating requests to + Amazon S3. + + Your account will need the following IAM policy to + upload to the cache: + + + + + Uploading with a specific credential profile for Amazon S3 + nix copy --to 's3://example-nix-cache?profile=cache-upload&region=eu-west-2' nixpkgs.hello + + + Uploading to an S3-Compatible Binary Cache + nix copy --to 's3://example-nix-cache?profile=cache-upload&scheme=https&endpoint=minio.example.com' nixpkgs.hello + +
+
diff --git a/third_party/nix/doc/manual/packages/sharing-packages.xml b/third_party/nix/doc/manual/packages/sharing-packages.xml new file mode 100644 index 000000000..bb6c52b8f --- /dev/null +++ b/third_party/nix/doc/manual/packages/sharing-packages.xml @@ -0,0 +1,20 @@ + + +Sharing Packages Between Machines + +Sometimes you want to copy a package from one machine to +another. Or, you want to install some packages and you know that +another machine already has some or all of those packages or their +dependencies. In that case there are mechanisms to quickly copy +packages between machines. + + + + + + + diff --git a/third_party/nix/doc/manual/packages/ssh-substituter.xml b/third_party/nix/doc/manual/packages/ssh-substituter.xml new file mode 100644 index 000000000..8db3f9662 --- /dev/null +++ b/third_party/nix/doc/manual/packages/ssh-substituter.xml @@ -0,0 +1,73 @@ +
+ +Serving a Nix store via SSH + +You can tell Nix to automatically fetch needed binaries from a +remote Nix store via SSH. For example, the following installs Firefox, +automatically fetching any store paths in Firefox’s closure if they +are available on the server avalon: + + +$ nix-env -i firefox --substituters ssh://alice@avalon + + +This works similar to the binary cache substituter that Nix usually +uses, only using SSH instead of HTTP: if a store path +P is needed, Nix will first check if it’s available +in the Nix store on avalon. If not, it will fall +back to using the binary cache substituter, and then to building from +source. + +The SSH substituter currently does not allow you to enter +an SSH passphrase interactively. Therefore, you should use +ssh-add to load the decrypted private key into +ssh-agent. + +You can also copy the closure of some store path, without +installing it into your profile, e.g. + + +$ nix-store -r /nix/store/m85bxg…-firefox-34.0.5 --substituters ssh://alice@avalon + + +This is essentially equivalent to doing + + +$ nix-copy-closure --from alice@avalon /nix/store/m85bxg…-firefox-34.0.5 + + + + +You can use SSH’s forced command feature to +set up a restricted user account for SSH substituter access, allowing +read-only access to the local Nix store, but nothing more. For +example, add the following lines to sshd_config +to restrict the user nix-ssh: + + +Match User nix-ssh + AllowAgentForwarding no + AllowTcpForwarding no + PermitTTY no + PermitTunnel no + X11Forwarding no + ForceCommand nix-store --serve +Match All + + +On NixOS, you can accomplish the same by adding the following to your +configuration.nix: + + +nix.sshServe.enable = true; +nix.sshServe.keys = [ "ssh-dss AAAAB3NzaC1k... bob@example.org" ]; + + +where the latter line lists the public keys of users that are allowed +to connect. + +
diff --git a/third_party/nix/doc/manual/quote-literals.xsl b/third_party/nix/doc/manual/quote-literals.xsl new file mode 100644 index 000000000..5002643db --- /dev/null +++ b/third_party/nix/doc/manual/quote-literals.xsl @@ -0,0 +1,40 @@ + + + + + + + `' + + + + + + + + + +
+ + + +
+ + + + + + + + + + + + + + + diff --git a/third_party/nix/doc/manual/release-notes/release-notes.xml b/third_party/nix/doc/manual/release-notes/release-notes.xml new file mode 100644 index 000000000..2655d68e3 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/release-notes.xml @@ -0,0 +1,51 @@ + + +Nix Release Notes + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/third_party/nix/doc/manual/release-notes/rl-0.10.1.xml b/third_party/nix/doc/manual/release-notes/rl-0.10.1.xml new file mode 100644 index 000000000..95829323d --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.10.1.xml @@ -0,0 +1,13 @@ +
+ +Release 0.10.1 (2006-10-11) + +This release fixes two somewhat obscure bugs that occur when +evaluating Nix expressions that are stored inside the Nix store +(NIX-67). These do not affect most users. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.10.xml b/third_party/nix/doc/manual/release-notes/rl-0.10.xml new file mode 100644 index 000000000..9afec4de9 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.10.xml @@ -0,0 +1,323 @@ +
+ +Release 0.10 (2006-10-06) + +This version of Nix uses Berkeley DB 4.4 instead of 4.3. +The database is upgraded automatically, but you should be careful not +to use old versions of Nix that still use Berkeley DB 4.3. In +particular, if you use a Nix installed through Nix, you should run + + +$ nix-store --clear-substitutes + +first. + +Also, the database schema has changed slighted to fix a +performance issue (see below). When you run any Nix 0.10 command for +the first time, the database will be upgraded automatically. This is +irreversible. + + + + + + + + nix-env usability improvements: + + + + An option + (or ) has been added to nix-env + --query to allow you to compare installed versions of + packages to available versions, or vice versa. An easy way to + see if you are up to date with what’s in your subscribed + channels is nix-env -qc \*. + + nix-env --query now takes as + arguments a list of package names about which to show + information, just like , etc.: for + example, nix-env -q gcc. Note that to show + all derivations, you need to specify + \*. + + nix-env -i + pkgname will now install + the highest available version of + pkgname, rather than installing all + available versions (which would probably give collisions) + (NIX-31). + + nix-env (-i|-u) --dry-run now + shows exactly which missing paths will be built or + substituted. + + nix-env -qa --description + shows human-readable descriptions of packages, provided that + they have a meta.description attribute (which + most packages in Nixpkgs don’t have yet). + + + + + + + New language features: + + + + Reference scanning (which happens after each + build) is much faster and takes a constant amount of + memory. + + String interpolation. Expressions like + + +"--with-freetype2-library=" + freetype + "/lib" + + can now be written as + + +"--with-freetype2-library=${freetype}/lib" + + You can write arbitrary expressions within + ${...}, not just + identifiers. + + Multi-line string literals. + + String concatenations can now involve + derivations, as in the example "--with-freetype2-library=" + + freetype + "/lib". This was not previously possible + because we need to register that a derivation that uses such a + string is dependent on freetype. The + evaluator now properly propagates this information. + Consequently, the subpath operator (~) has + been deprecated. + + Default values of function arguments can now + refer to other function arguments; that is, all arguments are in + scope in the default values + (NIX-45). + + + + Lots of new built-in primitives, such as + functions for list manipulation and integer arithmetic. See the + manual for a complete list. All primops are now available in + the set builtins, allowing one to test for + the availability of primop in a backwards-compatible + way. + + Real let-expressions: let x = ...; + ... z = ...; in .... + + + + + + + New commands nix-pack-closure and + nix-unpack-closure than can be used to easily + transfer a store path with all its dependencies to another machine. + Very convenient whenever you have some package on your machine and + you want to copy it somewhere else. + + + XML support: + + + + nix-env -q --xml prints the + installed or available packages in an XML representation for + easy processing by other tools. + + nix-instantiate --eval-only + --xml prints an XML representation of the resulting + term. (The new flag forces ‘deep’ + evaluation of the result, i.e., list elements and attributes are + evaluated recursively.) + + In Nix expressions, the primop + builtins.toXML converts a term to an XML + representation. This is primarily useful for passing structured + information to builders. + + + + + + + You can now unambiguously specify which derivation to + build or install in nix-env, + nix-instantiate and nix-build + using the / flags, which + takes an attribute name as argument. (Unlike symbolic package names + such as subversion-1.4.0, attribute names in an + attribute set are unique.) For instance, a quick way to perform a + test build of a package in Nixpkgs is nix-build + pkgs/top-level/all-packages.nix -A + foo. nix-env -q + --attr shows the attribute names corresponding to each + derivation. + + + If the top-level Nix expression used by + nix-env, nix-instantiate or + nix-build evaluates to a function whose arguments + all have default values, the function will be called automatically. + Also, the new command-line switch can be used to specify + function arguments on the command line. + + + nix-install-package --url + URL allows a package to be + installed directly from the given URL. + + + Nix now works behind an HTTP proxy server; just set + the standard environment variables http_proxy, + https_proxy, ftp_proxy or + all_proxy appropriately. Functions such as + fetchurl in Nixpkgs also respect these + variables. + + + nix-build -o + symlink allows the symlink to + the build result to be named something other than + result. + + + + + + Platform support: + + + + Support for 64-bit platforms, provided a suitably + patched ATerm library is used. Also, files larger than 2 + GiB are now supported. + + Added support for Cygwin (Windows, + i686-cygwin), Mac OS X on Intel + (i686-darwin) and Linux on PowerPC + (powerpc-linux). + + Users of SMP and multicore machines will + appreciate that the number of builds to be performed in parallel + can now be specified in the configuration file in the + build-max-jobs setting. + + + + + + + Garbage collector improvements: + + + + Open files (such as running programs) are now + used as roots of the garbage collector. This prevents programs + that have been uninstalled from being garbage collected while + they are still running. The script that detects these + additional runtime roots + (find-runtime-roots.pl) is inherently + system-specific, but it should work on Linux and on all + platforms that have the lsof + utility. + + nix-store --gc + (a.k.a. nix-collect-garbage) prints out the + number of bytes freed on standard output. nix-store + --gc --print-dead shows how many bytes would be freed + by an actual garbage collection. + + nix-collect-garbage -d + removes all old generations of all profiles + before calling the actual garbage collector (nix-store + --gc). This is an easy way to get rid of all old + packages in the Nix store. + + nix-store now has an + operation to delete specific paths + from the Nix store. It won’t delete reachable (non-garbage) + paths unless is + specified. + + + + + + + Berkeley DB 4.4’s process registry feature is used + to recover from crashed Nix processes. + + + + A performance issue has been fixed with the + referer table, which stores the inverse of the + references table (i.e., it tells you what store + paths refer to a given path). Maintaining this table could take a + quadratic amount of time, as well as a quadratic amount of Berkeley + DB log file space (in particular when running the garbage collector) + (NIX-23). + + Nix now catches the TERM and + HUP signals in addition to the + INT signal. So you can now do a killall + nix-store without triggering a database + recovery. + + bsdiff updated to version + 4.3. + + Substantial performance improvements in expression + evaluation and nix-env -qa, all thanks to Valgrind. Memory use has + been reduced by a factor 8 or so. Big speedup by memoisation of + path hashing. + + Lots of bug fixes, notably: + + + + Make sure that the garbage collector can run + successfully when the disk is full + (NIX-18). + + nix-env now locks the profile + to prevent races between concurrent nix-env + operations on the same profile + (NIX-7). + + Removed misleading messages from + nix-env -i (e.g., installing + `foo' followed by uninstalling + `foo') (NIX-17). + + + + + + Nix source distributions are a lot smaller now since + we no longer include a full copy of the Berkeley DB source + distribution (but only the bits we need). + + Header files are now installed so that external + programs can use the Nix libraries. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.11.xml b/third_party/nix/doc/manual/release-notes/rl-0.11.xml new file mode 100644 index 000000000..7ad0ab5b7 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.11.xml @@ -0,0 +1,261 @@ +
+ +Release 0.11 (2007-12-31) + +Nix 0.11 has many improvements over the previous stable release. +The most important improvement is secure multi-user support. It also +features many usability enhancements and language extensions, many of +them prompted by NixOS, the purely functional Linux distribution based +on Nix. Here is an (incomplete) list: + + + + + + Secure multi-user support. A single Nix store can + now be shared between multiple (possible untrusted) users. This is + an important feature for NixOS, where it allows non-root users to + install software. The old setuid method for sharing a store between + multiple users has been removed. Details for setting up a + multi-user store can be found in the manual. + + + The new command nix-copy-closure + gives you an easy and efficient way to exchange software between + machines. It copies the missing parts of the closure of a set of + store path to or from a remote machine via + ssh. + + + A new kind of string literal: strings between double + single-quotes ('') have indentation + “intelligently” removed. This allows large strings (such as shell + scripts or configuration file fragments in NixOS) to cleanly follow + the indentation of the surrounding expression. It also requires + much less escaping, since '' is less common in + most languages than ". + + + nix-env + modifies the current generation of a profile so that it contains + exactly the specified derivation, and nothing else. For example, + nix-env -p /nix/var/nix/profiles/browser --set + firefox lets the profile named + browser contain just Firefox. + + + nix-env now maintains + meta-information about installed packages in profiles. The + meta-information is the contents of the meta + attribute of derivations, such as description or + homepage. The command nix-env -q --xml + --meta shows all meta-information. + + + nix-env now uses the + meta.priority attribute of derivations to resolve + filename collisions between packages. Lower priority values denote + a higher priority. For instance, the GCC wrapper package and the + Binutils package in Nixpkgs both have a file + bin/ld, so previously if you tried to install + both you would get a collision. Now, on the other hand, the GCC + wrapper declares a higher priority than Binutils, so the former’s + bin/ld is symlinked in the user + environment. + + + nix-env -i / -u: instead of + breaking package ties by version, break them by priority and version + number. That is, if there are multiple packages with the same name, + then pick the package with the highest priority, and only use the + version if there are multiple packages with the same + priority. + + This makes it possible to mark specific versions/variant in + Nixpkgs more or less desirable than others. A typical example would + be a beta version of some package (e.g., + gcc-4.2.0rc1) which should not be installed even + though it is the highest version, except when it is explicitly + selected (e.g., nix-env -i + gcc-4.2.0rc1). + + + nix-env --set-flag allows meta + attributes of installed packages to be modified. There are several + attributes that can be usefully modified, because they affect the + behaviour of nix-env or the user environment + build script: + + + + meta.priority can be changed + to resolve filename clashes (see above). + + meta.keep can be set to + true to prevent the package from being + upgraded or replaced. Useful if you want to hang on to an older + version of a package. + + meta.active can be set to + false to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). + Set it back to true to re-enable the + package. + + + + + + + nix-env -q now has a flag + () that causes + nix-env to show only those derivations whose + output is already in the Nix store or that can be substituted (i.e., + downloaded from somewhere). In other words, it shows the packages + that can be installed “quickly”, i.e., don’t need to be built from + source. The flag is also available in + nix-env -i and nix-env -u to + filter out derivations for which no pre-built binary is + available. + + + The new option (in + nix-env, nix-instantiate and + nix-build) is like , except + that the value is a string. For example, --argstr system + i686-linux is equivalent to --arg system + \"i686-linux\" (note that + prevents annoying quoting around shell arguments). + + + nix-store has a new operation + () + paths that shows the build log of the given + paths. + + + + + + Nix now uses Berkeley DB 4.5. The database is + upgraded automatically, but you should be careful not to use old + versions of Nix that still use Berkeley DB 4.4. + + + + + + The option + (corresponding to the configuration setting + build-max-silent-time) allows you to set a + timeout on builds — if a build produces no output on + stdout or stderr for the given + number of seconds, it is terminated. This is useful for recovering + automatically from builds that are stuck in an infinite + loop. + + + nix-channel: each subscribed + channel is its own attribute in the top-level expression generated + for the channel. This allows disambiguation (e.g. nix-env + -i -A nixpkgs_unstable.firefox). + + + The substitutes table has been removed from the + database. This makes operations such as nix-pull + and nix-channel --update much, much + faster. + + + nix-pull now supports + bzip2-compressed manifests. This speeds up + channels. + + + nix-prefetch-url now has a + limited form of caching. This is used by + nix-channel to prevent unnecessary downloads when + the channel hasn’t changed. + + + nix-prefetch-url now by default + computes the SHA-256 hash of the file instead of the MD5 hash. In + calls to fetchurl you should pass the + sha256 attribute instead of + md5. You can pass either a hexadecimal or a + base-32 encoding of the hash. + + + Nix can now perform builds in an automatically + generated “chroot”. This prevents a builder from accessing stuff + outside of the Nix store, and thus helps ensure purity. This is an + experimental feature. + + + The new command nix-store + --optimise reduces Nix store disk space usage by finding + identical files in the store and hard-linking them to each other. + It typically reduces the size of the store by something like + 25-35%. + + + ~/.nix-defexpr can now be a + directory, in which case the Nix expressions in that directory are + combined into an attribute set, with the file names used as the + names of the attributes. The command nix-env + --import (which set the + ~/.nix-defexpr symlink) is + removed. + + + Derivations can specify the new special attribute + allowedReferences to enforce that the references + in the output of a derivation are a subset of a declared set of + paths. For example, if allowedReferences is an + empty list, then the output must not have any references. This is + used in NixOS to check that generated files such as initial ramdisks + for booting Linux don’t have any dependencies. + + + The new attribute + exportReferencesGraph allows builders access to + the references graph of their inputs. This is used in NixOS for + tasks such as generating ISO-9660 images that contain a Nix store + populated with the closure of certain paths. + + + Fixed-output derivations (like + fetchurl) can define the attribute + impureEnvVars to allow external environment + variables to be passed to builders. This is used in Nixpkgs to + support proxy configuration, among other things. + + + Several new built-in functions: + builtins.attrNames, + builtins.filterSource, + builtins.isAttrs, + builtins.isFunction, + builtins.listToAttrs, + builtins.stringLength, + builtins.sub, + builtins.substring, + throw, + builtins.trace, + builtins.readFile. + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.12.xml b/third_party/nix/doc/manual/release-notes/rl-0.12.xml new file mode 100644 index 000000000..fdba8c4d5 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.12.xml @@ -0,0 +1,175 @@ +
+ +Release 0.12 (2008-11-20) + + + + + Nix no longer uses Berkeley DB to store Nix store metadata. + The principal advantages of the new storage scheme are: it works + properly over decent implementations of NFS (allowing Nix stores + to be shared between multiple machines); no recovery is needed + when a Nix process crashes; no write access is needed for + read-only operations; no more running out of Berkeley DB locks on + certain operations. + + You still need to compile Nix with Berkeley DB support if + you want Nix to automatically convert your old Nix store to the + new schema. If you don’t need this, you can build Nix with the + configure option + . + + After the automatic conversion to the new schema, you can + delete the old Berkeley DB files: + + +$ cd /nix/var/nix/db +$ rm __db* log.* derivers references referrers reserved validpaths DB_CONFIG + + The new metadata is stored in the directories + /nix/var/nix/db/info and + /nix/var/nix/db/referrer. Though the + metadata is stored in human-readable plain-text files, they are + not intended to be human-editable, as Nix is rather strict about + the format. + + The new storage schema may or may not require less disk + space than the Berkeley DB environment, mostly depending on the + cluster size of your file system. With 1 KiB clusters (which + seems to be the ext3 default nowadays) it + usually takes up much less space. + + + There is a new substituter that copies paths + directly from other (remote) Nix stores mounted somewhere in the + filesystem. For instance, you can speed up an installation by + mounting some remote Nix store that already has the packages in + question via NFS or sshfs. The environment + variable NIX_OTHER_STORES specifies the locations of + the remote Nix directories, + e.g. /mnt/remote-fs/nix. + + New nix-store operations + and to dump + and reload the Nix database. + + The garbage collector has a number of new options to + allow only some of the garbage to be deleted. The option + tells the + collector to stop after at least N bytes + have been deleted. The option tells it to stop after the + link count on /nix/store has dropped below + N. This is useful for very large Nix + stores on filesystems with a 32000 subdirectories limit (like + ext3). The option + causes store paths to be deleted in order of ascending last access + time. This allows non-recently used stuff to be deleted. The + option + specifies an upper limit to the last accessed time of paths that may + be deleted. For instance, + + + $ nix-store --gc -v --max-atime $(date +%s -d "2 months ago") + + deletes everything that hasn’t been accessed in two months. + + nix-env now uses optimistic + profile locking when performing an operation like installing or + upgrading, instead of setting an exclusive lock on the profile. + This allows multiple nix-env -i / -u / -e + operations on the same profile in parallel. If a + nix-env operation sees at the end that the profile + was changed in the meantime by another process, it will just + restart. This is generally cheap because the build results are + still in the Nix store. + + The option is now + supported by nix-store -r and + nix-build. + + The information previously shown by + (i.e., which derivations will be built + and which paths will be substituted) is now always shown by + nix-env, nix-store -r and + nix-build. The total download size of + substitutable paths is now also shown. For instance, a build will + show something like + + +the following derivations will be built: + /nix/store/129sbxnk5n466zg6r1qmq1xjv9zymyy7-activate-configuration.sh.drv + /nix/store/7mzy971rdm8l566ch8hgxaf89x7lr7ik-upstart-jobs.drv + ... +the following paths will be downloaded/copied (30.02 MiB): + /nix/store/4m8pvgy2dcjgppf5b4cj5l6wyshjhalj-samba-3.2.4 + /nix/store/7h1kwcj29ip8vk26rhmx6bfjraxp0g4l-libunwind-0.98.6 + ... + + + + Language features: + + + + @-patterns as in Haskell. For instance, in a + function definition + + f = args @ {x, y, z}: ...; + + args refers to the argument as a whole, which + is further pattern-matched against the attribute set pattern + {x, y, z}. + + “...” (ellipsis) patterns. + An attribute set pattern can now say ... at + the end of the attribute name list to specify that the function + takes at least the listed attributes, while + ignoring additional attributes. For instance, + + {stdenv, fetchurl, fuse, ...}: ... + + defines a function that accepts any attribute set that includes + at least the three listed attributes. + + New primops: + builtins.parseDrvName (split a package name + string like "nix-0.12pre12876" into its name + and version components, e.g. "nix" and + "0.12pre12876"), + builtins.compareVersions (compare two version + strings using the same algorithm that nix-env + uses), builtins.length (efficiently compute + the length of a list), builtins.mul (integer + multiplication), builtins.div (integer + division). + + + + + + + + nix-prefetch-url now supports + mirror:// URLs, provided that the environment + variable NIXPKGS_ALL points at a Nixpkgs + tree. + + Removed the commands + nix-pack-closure and + nix-unpack-closure. You can do almost the same + thing but much more efficiently by doing nix-store --export + $(nix-store -qR paths) > closure and + nix-store --import < + closure. + + Lots of bug fixes, including a big performance bug in + the handling of with-expressions. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.13.xml b/third_party/nix/doc/manual/release-notes/rl-0.13.xml new file mode 100644 index 000000000..cce2e4a26 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.13.xml @@ -0,0 +1,106 @@ +
+ +Release 0.13 (2009-11-05) + +This is primarily a bug fix release. It has some new +features: + + + + + Syntactic sugar for writing nested attribute sets. Instead of + + +{ + foo = { + bar = 123; + xyzzy = true; + }; + a = { b = { c = "d"; }; }; +} + + + you can write + + +{ + foo.bar = 123; + foo.xyzzy = true; + a.b.c = "d"; +} + + + This is useful, for instance, in NixOS configuration files. + + + + + Support for Nix channels generated by Hydra, the Nix-based + continuous build system. (Hydra generates NAR archives on the + fly, so the size and hash of these archives isn’t known in + advance.) + + + + Support i686-linux builds directly on + x86_64-linux Nix installations. This is + implemented using the personality() syscall, + which causes uname to return + i686 in child processes. + + + + Various improvements to the chroot + support. Building in a chroot works quite well + now. + + + + Nix no longer blocks if it tries to build a path and another + process is already building the same path. Instead it tries to + build another buildable path first. This improves + parallelism. + + + + Support for large (> 4 GiB) files in NAR archives. + + + + Various (performance) improvements to the remote build + mechanism. + + + + New primops: builtins.addErrorContext (to + add a string to stack traces — useful for debugging), + builtins.isBool, + builtins.isString, + builtins.isInt, + builtins.intersectAttrs. + + + + OpenSolaris support (Sander van der Burg). + + + + Stack traces are no longer displayed unless the + option is used. + + + + The scoping rules for inherit + (e) ... in recursive + attribute sets have changed. The expression + e can now refer to the attributes + defined in the containing set. + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.14.xml b/third_party/nix/doc/manual/release-notes/rl-0.14.xml new file mode 100644 index 000000000..e5fe9da78 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.14.xml @@ -0,0 +1,46 @@ +
+ +Release 0.14 (2010-02-04) + +This release has the following improvements: + + + + + The garbage collector now starts deleting garbage much + faster than before. It no longer determines liveness of all paths + in the store, but does so on demand. + + + + Added a new operation, nix-store --query + --roots, that shows the garbage collector roots that + directly or indirectly point to the given store paths. + + + + Removed support for converting Berkeley DB-based Nix + databases to the new schema. + + + + Removed the and + garbage collector options. They were + not very useful in practice. + + + + On Windows, Nix now requires Cygwin 1.7.x. + + + + A few bug fixes. + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.15.xml b/third_party/nix/doc/manual/release-notes/rl-0.15.xml new file mode 100644 index 000000000..9f58a8efc --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.15.xml @@ -0,0 +1,14 @@ +
+ +Release 0.15 (2010-03-17) + +This is a bug-fix release. Among other things, it fixes +building on Mac OS X (Snow Leopard), and improves the contents of +/etc/passwd and /etc/group +in chroot builds. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.16.xml b/third_party/nix/doc/manual/release-notes/rl-0.16.xml new file mode 100644 index 000000000..af1edc0eb --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.16.xml @@ -0,0 +1,55 @@ +
+ +Release 0.16 (2010-08-17) + +This release has the following improvements: + + + + + The Nix expression evaluator is now much faster in most + cases: typically, 3 + to 8 times compared to the old implementation. It also + uses less memory. It no longer depends on the ATerm + library. + + + + + Support for configurable parallelism inside builders. Build + scripts have always had the ability to perform multiple build + actions in parallel (for instance, by running make -j + 2), but this was not desirable because the number of + actions to be performed in parallel was not configurable. Nix + now has an option as well as a configuration + setting build-cores = + N that causes the + environment variable NIX_BUILD_CORES to be set to + N when the builder is invoked. The + builder can use this at its discretion to perform a parallel + build, e.g., by calling make -j + N. In Nixpkgs, this can be + enabled on a per-package basis by setting the derivation + attribute enableParallelBuilding to + true. + + + + + nix-store -q now supports XML output + through the flag. + + + + Several bug fixes. + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.5.xml b/third_party/nix/doc/manual/release-notes/rl-0.5.xml new file mode 100644 index 000000000..e9f8bf270 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.5.xml @@ -0,0 +1,11 @@ +
+ +Release 0.5 and earlier + +Please refer to the Subversion commit log messages. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.6.xml b/third_party/nix/doc/manual/release-notes/rl-0.6.xml new file mode 100644 index 000000000..6dc6521d3 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.6.xml @@ -0,0 +1,122 @@ +
+ +Release 0.6 (2004-11-14) + + + + + Rewrite of the normalisation engine. + + + + Multiple builds can now be performed in parallel + (option ). + + Distributed builds. Nix can now call a shell + script to forward builds to Nix installations on remote + machines, which may or may not be of the same platform + type. + + Option allows + recovery from broken substitutes. + + Option causes + building of other (unaffected) derivations to continue if one + failed. + + + + + + + + Improvements to the garbage collector (i.e., it + should actually work now). + + Setuid Nix installations allow a Nix store to be + shared among multiple users. + + Substitute registration is much faster + now. + + A utility nix-build to build a + Nix expression and create a symlink to the result int the current + directory; useful for testing Nix derivations. + + Manual updates. + + + + nix-env changes: + + + + Derivations for other platforms are filtered out + (which can be overridden using + ). + + by default now + uninstall previous derivations with the same + name. + + allows upgrading to a + specific version. + + New operation + to remove profile + generations (necessary for effective garbage + collection). + + Nicer output (sorted, + columnised). + + + + + + + + More sensible verbosity levels all around (builder + output is now shown always, unless is + given). + + + + Nix expression language changes: + + + + New language construct: with + E1; + E2 brings all attributes + defined in the attribute set E1 in + scope in E2. + + Added a map + function. + + Various new operators (e.g., string + concatenation). + + + + + + + + Expression evaluation is much + faster. + + An Emacs mode for editing Nix expressions (with + syntax highlighting and indentation) has been + added. + + Many bug fixes. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.7.xml b/third_party/nix/doc/manual/release-notes/rl-0.7.xml new file mode 100644 index 000000000..6f95db436 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.7.xml @@ -0,0 +1,35 @@ +
+ +Release 0.7 (2005-01-12) + + + + Binary patching. When upgrading components using + pre-built binaries (through nix-pull / nix-channel), Nix can + automatically download and apply binary patches to already installed + components instead of full downloads. Patching is “smart”: if there + is a sequence of patches to an installed + component, Nix will use it. Patches are currently generated + automatically between Nixpkgs (pre-)releases. + + Simplifications to the substitute + mechanism. + + Nix-pull now stores downloaded manifests in + /nix/var/nix/manifests. + + Metadata on files in the Nix store is canonicalised + after builds: the last-modified timestamp is set to 0 (00:00:00 + 1/1/1970), the mode is set to 0444 or 0555 (readable and possibly + executable by all; setuid/setgid bits are dropped), and the group is + set to the default. This ensures that the result of a build and an + installation through a substitute is the same; and that timestamp + dependencies are revealed. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.8.1.xml b/third_party/nix/doc/manual/release-notes/rl-0.8.1.xml new file mode 100644 index 000000000..f7ffca0f8 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.8.1.xml @@ -0,0 +1,21 @@ +
+ +Release 0.8.1 (2005-04-13) + +This is a bug fix release. + + + + Patch downloading was broken. + + The garbage collector would not delete paths that + had references from invalid (but substitutable) + paths. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.8.xml b/third_party/nix/doc/manual/release-notes/rl-0.8.xml new file mode 100644 index 000000000..784b26c6b --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.8.xml @@ -0,0 +1,246 @@ +
+ +Release 0.8 (2005-04-11) + +NOTE: the hashing scheme in Nix 0.8 changed (as detailed below). +As a result, nix-pull manifests and channels built +for Nix 0.7 and below will now work anymore. However, the Nix +expression language has not changed, so you can still build from +source. Also, existing user environments continue to work. Nix 0.8 +will automatically upgrade the database schema of previous +installations when it is first run. + +If you get the error message + + +you have an old-style manifest `/nix/var/nix/manifests/[...]'; please +delete it + +you should delete previously downloaded manifests: + + +$ rm /nix/var/nix/manifests/* + +If nix-channel gives the error message + + +manifest `http://catamaran.labs.cs.uu.nl/dist/nix/channels/[channel]/MANIFEST' +is too old (i.e., for Nix <= 0.7) + +then you should unsubscribe from the offending channel +(nix-channel --remove +URL; leave out +/MANIFEST), and subscribe to the same URL, with +channels replaced by channels-v3 +(e.g., ). + +Nix 0.8 has the following improvements: + + + + The cryptographic hashes used in store paths are now + 160 bits long, but encoded in base-32 so that they are still only 32 + characters long (e.g., + /nix/store/csw87wag8bqlqk7ipllbwypb14xainap-atk-1.9.0). + (This is actually a 160 bit truncation of a SHA-256 + hash.) + + Big cleanups and simplifications of the basic store + semantics. The notion of “closure store expressions” is gone (and + so is the notion of “successors”); the file system references of a + store path are now just stored in the database. + + For instance, given any store path, you can query its closure: + + +$ nix-store -qR $(which firefox) +... lots of paths ... + + Also, Nix now remembers for each store path the derivation that + built it (the “deriver”): + + +$ nix-store -qR $(which firefox) +/nix/store/4b0jx7vq80l9aqcnkszxhymsf1ffa5jd-firefox-1.0.1.drv + + So to see the build-time dependencies, you can do + + +$ nix-store -qR $(nix-store -qd $(which firefox)) + + or, in a nicer format: + + +$ nix-store -q --tree $(nix-store -qd $(which firefox)) + + + + File system references are also stored in reverse. For + instance, you can query all paths that directly or indirectly use a + certain Glibc: + + +$ nix-store -q --referrers-closure \ + /nix/store/8lz9yc6zgmc0vlqmn2ipcpkjlmbi51vv-glibc-2.3.4 + + + + + + The concept of fixed-output derivations has been + formalised. Previously, functions such as + fetchurl in Nixpkgs used a hack (namely, + explicitly specifying a store path hash) to prevent changes to, say, + the URL of the file from propagating upwards through the dependency + graph, causing rebuilds of everything. This can now be done cleanly + by specifying the outputHash and + outputHashAlgo attributes. Nix itself checks + that the content of the output has the specified hash. (This is + important for maintaining certain invariants necessary for future + work on secure shared stores.) + + One-click installation :-) It is now possible to + install any top-level component in Nixpkgs directly, through the web + — see, e.g., . + All you have to do is associate + /nix/bin/nix-install-package with the MIME type + application/nix-package (or the extension + .nixpkg), and clicking on a package link will + cause it to be installed, with all appropriate dependencies. If you + just want to install some specific application, this is easier than + subscribing to a channel. + + nix-store -r + PATHS now builds all the + derivations PATHS in parallel. Previously it did them sequentially + (though exploiting possible parallelism between subderivations). + This is nice for build farms. + + nix-channel has new operations + and + . + + New ways of installing components into user + environments: + + + + Copy from another user environment: + + +$ nix-env -i --from-profile .../other-profile firefox + + + + Install a store derivation directly (bypassing the + Nix expression language entirely): + + +$ nix-env -i /nix/store/z58v41v21xd3...-aterm-2.3.1.drv + + (This is used to implement nix-install-package, + which is therefore immune to evolution in the Nix expression + language.) + + Install an already built store path directly: + + +$ nix-env -i /nix/store/hsyj5pbn0d9i...-aterm-2.3.1 + + + + Install the result of a Nix expression specified + as a command-line argument: + + +$ nix-env -f .../i686-linux.nix -i -E 'x: x.firefoxWrapper' + + The difference with the normal installation mode is that + does not use the name + attributes of derivations. Therefore, this can be used to + disambiguate multiple derivations with the same + name. + + + + A hash of the contents of a store path is now stored + in the database after a successful build. This allows you to check + whether store paths have been tampered with: nix-store + --verify --check-contents. + + + + Implemented a concurrent garbage collector. It is now + always safe to run the garbage collector, even if other Nix + operations are happening simultaneously. + + However, there can still be GC races if you use + nix-instantiate and nix-store + --realise directly to build things. To prevent races, + use the flag of those commands. + + + + The garbage collector now finally deletes paths in + the right order (i.e., topologically sorted under the “references” + relation), thus making it safe to interrupt the collector without + risking a store that violates the closure + invariant. + + Likewise, the substitute mechanism now downloads + files in the right order, thus preserving the closure invariant at + all times. + + The result of nix-build is now + registered as a root of the garbage collector. If the + ./result link is deleted, the GC root + disappears automatically. + + + + The behaviour of the garbage collector can be changed + globally by setting options in + /nix/etc/nix/nix.conf. + + + + gc-keep-derivations specifies + whether deriver links should be followed when searching for live + paths. + + gc-keep-outputs specifies + whether outputs of derivations should be followed when searching + for live paths. + + env-keep-derivations + specifies whether user environments should store the paths of + derivations when they are added (thus keeping the derivations + alive). + + + + + + New nix-env query flags + and + . + + fetchurl allows SHA-1 and SHA-256 + in addition to MD5. Just specify the attribute + sha1 or sha256 instead of + md5. + + Manual updates. + + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.9.1.xml b/third_party/nix/doc/manual/release-notes/rl-0.9.1.xml new file mode 100644 index 000000000..85d11f416 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.9.1.xml @@ -0,0 +1,13 @@ +
+ +Release 0.9.1 (2005-09-20) + +This bug fix release addresses a problem with the ATerm library +when the flag in +configure was not used. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.9.2.xml b/third_party/nix/doc/manual/release-notes/rl-0.9.2.xml new file mode 100644 index 000000000..cb705e98a --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.9.2.xml @@ -0,0 +1,28 @@ +
+ +Release 0.9.2 (2005-09-21) + +This bug fix release fixes two problems on Mac OS X: + + + + If Nix was linked against statically linked versions + of the ATerm or Berkeley DB library, there would be dynamic link + errors at runtime. + + nix-pull and + nix-push intermittently failed due to race + conditions involving pipes and child processes with error messages + such as open2: open(GLOB(0x180b2e4), >&=9) failed: Bad + file descriptor at /nix/bin/nix-pull line 77 (issue + NIX-14). + + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-0.9.xml b/third_party/nix/doc/manual/release-notes/rl-0.9.xml new file mode 100644 index 000000000..fd1e633f7 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-0.9.xml @@ -0,0 +1,98 @@ +
+ +Release 0.9 (2005-09-16) + +NOTE: this version of Nix uses Berkeley DB 4.3 instead of 4.2. +The database is upgraded automatically, but you should be careful not +to use old versions of Nix that still use Berkeley DB 4.2. In +particular, if you use a Nix installed through Nix, you should run + + +$ nix-store --clear-substitutes + +first. + + + + + Unpacking of patch sequences is much faster now + since we no longer do redundant unpacking and repacking of + intermediate paths. + + Nix now uses Berkeley DB 4.3. + + The derivation primitive is + lazier. Attributes of dependent derivations can mutually refer to + each other (as long as there are no data dependencies on the + outPath and drvPath attributes + computed by derivation). + + For example, the expression derivation + attrs now evaluates to (essentially) + + +attrs // { + type = "derivation"; + outPath = derivation! attrs; + drvPath = derivation! attrs; +} + + where derivation! is a primop that does the + actual derivation instantiation (i.e., it does what + derivation used to do). The advantage is that + it allows commands such as nix-env -qa and + nix-env -i to be much faster since they no longer + need to instantiate all derivations, just the + name attribute. + + Also, it allows derivations to cyclically reference each + other, for example, + + +webServer = derivation { + ... + hostName = "svn.cs.uu.nl"; + services = [svnService]; +}; + +svnService = derivation { + ... + hostName = webServer.hostName; +}; + + Previously, this would yield a black hole (infinite recursion). + + + + nix-build now defaults to using + ./default.nix if no Nix expression is + specified. + + nix-instantiate, when applied to + a Nix expression that evaluates to a function, will call the + function automatically if all its arguments have + defaults. + + Nix now uses libtool to build dynamic libraries. + This reduces the size of executables. + + A new list concatenation operator + ++. For example, [1 2 3] ++ [4 5 + 6] evaluates to [1 2 3 4 5 + 6]. + + Some currently undocumented primops to support + low-level build management using Nix (i.e., using Nix as a Make + replacement). See the commit messages for r3578 + and r3580. + + Various bug fixes and performance + improvements. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.0.xml b/third_party/nix/doc/manual/release-notes/rl-1.0.xml new file mode 100644 index 000000000..ff11168d0 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.0.xml @@ -0,0 +1,119 @@ +
+ +Release 1.0 (2012-05-11) + +There have been numerous improvements and bug fixes since the +previous release. Here are the most significant: + + + + + Nix can now optionally use the Boehm garbage collector. + This significantly reduces the Nix evaluator’s memory footprint, + especially when evaluating large NixOS system configurations. It + can be enabled using the configure + option. + + + + Nix now uses SQLite for its database. This is faster and + more flexible than the old ad hoc format. + SQLite is also used to cache the manifests in + /nix/var/nix/manifests, resulting in a + significant speedup. + + + + Nix now has an search path for expressions. The search path + is set using the environment variable NIX_PATH and + the command line option. In Nix expressions, + paths between angle brackets are used to specify files that must + be looked up in the search path. For instance, the expression + <nixpkgs/default.nix> looks for a file + nixpkgs/default.nix relative to every element + in the search path. + + + + The new command nix-build --run-env + builds all dependencies of a derivation, then starts a shell in an + environment containing all variables from the derivation. This is + useful for reproducing the environment of a derivation for + development. + + + + The new command nix-store --verify-path + verifies that the contents of a store path have not + changed. + + + + The new command nix-store --print-env + prints out the environment of a derivation in a format that can be + evaluated by a shell. + + + + Attribute names can now be arbitrary strings. For instance, + you can write { "foo-1.2" = …; "bla bla" = …; }."bla + bla". + + + + Attribute selection can now provide a default value using + the or operator. For instance, the expression + x.y.z or e evaluates to the attribute + x.y.z if it exists, and e + otherwise. + + + + The right-hand side of the ? operator can + now be an attribute path, e.g., attrs ? + a.b.c. + + + + On Linux, Nix will now make files in the Nix store immutable + on filesystems that support it. This prevents accidental + modification of files in the store by the root user. + + + + Nix has preliminary support for derivations with multiple + outputs. This is useful because it allows parts of a package to + be deployed and garbage-collected separately. For instance, + development parts of a package such as header files or static + libraries would typically not be part of the closure of an + application, resulting in reduced disk usage and installation + time. + + + + The Nix store garbage collector is faster and holds the + global lock for a shorter amount of time. + + + + The option (corresponding to the + configuration setting build-timeout) allows you + to set an absolute timeout on builds — if a build runs for more than + the given number of seconds, it is terminated. This is useful for + recovering automatically from builds that are stuck in an infinite + loop but keep producing output, and for which + --max-silent-time is ineffective. + + + + Nix development has moved to GitHub (). + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.1.xml b/third_party/nix/doc/manual/release-notes/rl-1.1.xml new file mode 100644 index 000000000..2f26e7a24 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.1.xml @@ -0,0 +1,100 @@ +
+ +Release 1.1 (2012-07-18) + +This release has the following improvements: + + + + + On Linux, when doing a chroot build, Nix now uses various + namespace features provided by the Linux kernel to improve + build isolation. Namely: + + The private network namespace ensures that + builders cannot talk to the outside world (or vice versa): each + build only sees a private loopback interface. This also means + that two concurrent builds can listen on the same port (e.g. as + part of a test) without conflicting with each + other. + The PID namespace causes each build to start as + PID 1. Processes outside of the chroot are not visible to those + on the inside. On the other hand, processes inside the chroot + are visible from the outside (though with + different PIDs). + The IPC namespace prevents the builder from + communicating with outside processes using SysV IPC mechanisms + (shared memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + The UTS namespace ensures that builders see a + hostname of localhost rather than the actual + hostname. + The private mount namespace was already used by + Nix to ensure that the bind-mounts used to set up the chroot are + cleaned up automatically. + + + + + + Build logs are now compressed using + bzip2. The command nix-store + -l decompresses them on the fly. This can be disabled + by setting the option build-compress-log to + false. + + + + The creation of build logs in + /nix/var/log/nix/drvs can be disabled by + setting the new option build-keep-log to + false. This is useful, for instance, for Hydra + build machines. + + + + Nix now reserves some space in + /nix/var/nix/db/reserved to ensure that the + garbage collector can run successfully if the disk is full. This + is necessary because SQLite transactions fail if the disk is + full. + + + + Added a basic fetchurl function. This + is not intended to replace the fetchurl in + Nixpkgs, but is useful for bootstrapping; e.g., it will allow us + to get rid of the bootstrap binaries in the Nixpkgs source tree + and download them instead. You can use it by doing + import <nix/fetchurl.nix> { url = + url; sha256 = + "hash"; }. (Shea Levy) + + + + Improved RPM spec file. (Michel Alexandre Salim) + + + + Support for on-demand socket-based activation in the Nix + daemon with systemd. + + + + Added a manpage for + nix.conf5. + + + + When using the Nix daemon, the flag in + nix-env -qa is now much faster. + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.10.xml b/third_party/nix/doc/manual/release-notes/rl-1.10.xml new file mode 100644 index 000000000..689a95466 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.10.xml @@ -0,0 +1,64 @@ +
+ +Release 1.10 (2015-09-03) + +This is primarily a bug fix release. It also has a number of new +features: + + + + + A number of builtin functions have been added to reduce + Nixpkgs/NixOS evaluation time and memory consumption: + all, + any, + concatStringsSep, + foldl’, + genList, + replaceStrings, + sort. + + + + + The garbage collector is more robust when the disk is full. + + + + Nix supports a new API for building derivations that doesn’t + require a .drv file to be present on disk; it + only requires an in-memory representation of the derivation. This + is used by the Hydra continuous build system to make remote builds + more efficient. + + + + The function <nix/fetchurl.nix> now + uses a builtin builder (i.e. it doesn’t + require starting an external process; the download is performed by + Nix itself). This ensures that derivation paths don’t change when + Nix is upgraded, and obviates the need for ugly hacks to support + chroot execution. + + + + now prints some configuration + information, in particular what compile-time optional features are + enabled, and the paths of various directories. + + + + Build users have their supplementary groups set correctly. + + + + +This release has contributions from Eelco Dolstra, Guillaume +Maudoux, Iwan Aucamp, Jaka Hudoklin, Kirill Elagin, Ludovic Courtès, +Manolis Ragkousis, Nicolas B. Pierron and Shea Levy. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.11.10.xml b/third_party/nix/doc/manual/release-notes/rl-1.11.10.xml new file mode 100644 index 000000000..415388b3e --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.11.10.xml @@ -0,0 +1,31 @@ +
+ +Release 1.11.10 (2017-06-12) + +This release fixes a security bug in Nix’s “build user” build +isolation mechanism. Previously, Nix builders had the ability to +create setuid binaries owned by a nixbld +user. Such a binary could then be used by an attacker to assume a +nixbld identity and interfere with subsequent +builds running under the same UID. + +To prevent this issue, Nix now disallows builders to create +setuid and setgid binaries. On Linux, this is done using a seccomp BPF +filter. Note that this imposes a small performance penalty (e.g. 1% +when building GNU Hello). Using seccomp, we now also prevent the +creation of extended attributes and POSIX ACLs since these cannot be +represented in the NAR format and (in the case of POSIX ACLs) allow +bypassing regular Nix store permissions. On macOS, the restriction is +implemented using the existing sandbox mechanism, which now uses a +minimal “allow all except the creation of setuid/setgid binaries” +profile when regular sandboxing is disabled. On other platforms, the +“build user” mechanism is now disabled. + +Thanks go to Linus Heckemann for discovering and reporting this +bug. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.11.xml b/third_party/nix/doc/manual/release-notes/rl-1.11.xml new file mode 100644 index 000000000..fe422dd1f --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.11.xml @@ -0,0 +1,141 @@ +
+ +Release 1.11 (2016-01-19) + +This is primarily a bug fix release. It also has a number of new +features: + + + + + nix-prefetch-url can now download URLs + specified in a Nix expression. For example, + + +$ nix-prefetch-url -A hello.src + + + will prefetch the file specified by the + fetchurl call in the attribute + hello.src from the Nix expression in the + current directory, and print the cryptographic hash of the + resulting file on stdout. This differs from nix-build -A + hello.src in that it doesn't verify the hash, and is + thus useful when you’re updating a Nix expression. + + You can also prefetch the result of functions that unpack a + tarball, such as fetchFromGitHub. For example: + + +$ nix-prefetch-url --unpack https://github.com/NixOS/patchelf/archive/0.8.tar.gz + + + or from a Nix expression: + + +$ nix-prefetch-url -A nix-repl.src + + + + + + + + The builtin function + <nix/fetchurl.nix> now supports + downloading and unpacking NARs. This removes the need to have + multiple downloads in the Nixpkgs stdenv bootstrap process (like a + separate busybox binary for Linux, or curl/mkdir/sh/bzip2 for + Darwin). Now all those files can be combined into a single NAR, + optionally compressed using xz. + + + + Nix now supports SHA-512 hashes for verifying fixed-output + derivations, and in builtins.hashString. + + + + + The new flag will cause every build to + be executed N+1 times. If the build + output differs between any round, the build is rejected, and the + output paths are not registered as valid. This is primarily + useful to verify build determinism. (We already had a + option to repeat a previously succeeded + build. However, with , non-deterministic + builds are registered in the DB. Preventing that is useful for + Hydra to ensure that non-deterministic builds don't end up + getting published to the binary cache.) + + + + + + The options and , if they + detect a difference between two runs of the same derivation and + is given, will make the output of the other + run available under + store-path-check. This + makes it easier to investigate the non-determinism using tools + like diffoscope, e.g., + + +$ nix-build pkgs/stdenv/linux -A stage1.pkgs.zlib --check -K +error: derivation ‘/nix/store/l54i8wlw2265…-zlib-1.2.8.drv’ may not +be deterministic: output ‘/nix/store/11a27shh6n2i…-zlib-1.2.8’ +differs from ‘/nix/store/11a27shh6n2i…-zlib-1.2.8-check’ + +$ diffoscope /nix/store/11a27shh6n2i…-zlib-1.2.8 /nix/store/11a27shh6n2i…-zlib-1.2.8-check +… +├── lib/libz.a +│ ├── metadata +│ │ @@ -1,15 +1,15 @@ +│ │ -rw-r--r-- 30001/30000 3096 Jan 12 15:20 2016 adler32.o +… +│ │ +rw-r--r-- 30001/30000 3096 Jan 12 15:28 2016 adler32.o +… + + + + + + Improved FreeBSD support. + + + + nix-env -qa --xml --meta now prints + license information. + + + + The maximum number of parallel TCP connections that the + binary cache substituter will use has been decreased from 150 to + 25. This should prevent upsetting some broken NAT routers, and + also improves performance. + + + + All "chroot"-containing strings got renamed to "sandbox". + In particular, some Nix options got renamed, but the old names + are still accepted as lower-priority aliases. + + + + + +This release has contributions from Anders Claesson, Anthony +Cowley, Bjørn Forsman, Brian McKenna, Danny Wilson, davidak, Eelco Dolstra, +Fabian Schmitthenner, FrankHB, Ilya Novoselov, janus, Jim Garrison, John +Ericson, Jude Taylor, Ludovic Courtès, Manuel Jacob, Mathnerd314, +Pascal Wittmann, Peter Simons, Philip Potter, Preston Bennes, Rommel +M. Martinez, Sander van der Burg, Shea Levy, Tim Cuthbertson, Tuomas +Tynkkynen, Utku Demir and Vladimír Čunát. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.2.xml b/third_party/nix/doc/manual/release-notes/rl-1.2.xml new file mode 100644 index 000000000..748fd9e67 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.2.xml @@ -0,0 +1,157 @@ +
+ +Release 1.2 (2012-12-06) + +This release has the following improvements and changes: + + + + + Nix has a new binary substituter mechanism: the + binary cache. A binary cache contains + pre-built binaries of Nix packages. Whenever Nix wants to build a + missing Nix store path, it will check a set of binary caches to + see if any of them has a pre-built binary of that path. The + configuration setting contains a + list of URLs of binary caches. For instance, doing + +$ nix-env -i thunderbird --option binary-caches http://cache.nixos.org + + will install Thunderbird and its dependencies, using the available + pre-built binaries in http://cache.nixos.org. + The main advantage over the old “manifest”-based method of getting + pre-built binaries is that you don’t have to worry about your + manifest being in sync with the Nix expressions you’re installing + from; i.e., you don’t need to run nix-pull to + update your manifest. It’s also more scalable because you don’t + need to redownload a giant manifest file every time. + + + A Nix channel can provide a binary cache URL that will be + used automatically if you subscribe to that channel. If you use + the Nixpkgs or NixOS channels + (http://nixos.org/channels) you automatically get the + cache http://cache.nixos.org. + + Binary caches are created using nix-push. + For details on the operation and format of binary caches, see the + nix-push manpage. More details are provided in + this + nix-dev posting. + + + + Multiple output support should now be usable. A derivation + can declare that it wants to produce multiple store paths by + saying something like + +outputs = [ "lib" "headers" "doc" ]; + + This will cause Nix to pass the intended store path of each output + to the builder through the environment variables + lib, headers and + doc. Other packages can refer to a specific + output by referring to + pkg.output, + e.g. + +buildInputs = [ pkg.lib pkg.headers ]; + + If you install a package with multiple outputs using + nix-env, each output path will be symlinked + into the user environment. + + + + Dashes are now valid as part of identifiers and attribute + names. + + + + The new operation nix-store --repair-path + allows corrupted or missing store paths to be repaired by + redownloading them. nix-store --verify --check-contents + --repair will scan and repair all paths in the Nix + store. Similarly, nix-env, + nix-build, nix-instantiate + and nix-store --realise have a + flag to detect and fix bad paths by + rebuilding or redownloading them. + + + + Nix no longer sets the immutable bit on files in the Nix + store. Instead, the recommended way to guard the Nix store + against accidental modification on Linux is to make it a read-only + bind mount, like this: + + +$ mount --bind /nix/store /nix/store +$ mount -o remount,ro,bind /nix/store + + + Nix will automatically make /nix/store + writable as needed (using a private mount namespace) to allow + modifications. + + + + Store optimisation (replacing identical files in the store + with hard links) can now be done automatically every time a path + is added to the store. This is enabled by setting the + configuration option auto-optimise-store to + true (disabled by default). + + + + Nix now supports xz compression for NARs + in addition to bzip2. It compresses about 30% + better on typical archives and decompresses about twice as + fast. + + + + Basic Nix expression evaluation profiling: setting the + environment variable NIX_COUNT_CALLS to + 1 will cause Nix to print how many times each + primop or function was executed. + + + + New primops: concatLists, + elem, elemAt and + filter. + + + + The command nix-copy-closure has a new + flag () to + download missing paths on the target machine using the substitute + mechanism. + + + + The command nix-worker has been renamed + to nix-daemon. Support for running the Nix + worker in “slave” mode has been removed. + + + + The flag of every Nix command now + invokes man. + + + + Chroot builds are now supported on systemd machines. + + + + +This release has contributions from Eelco Dolstra, Florian +Friesdorf, Mats Erik Andersson and Shea Levy. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.3.xml b/third_party/nix/doc/manual/release-notes/rl-1.3.xml new file mode 100644 index 000000000..e2009ee3b --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.3.xml @@ -0,0 +1,19 @@ +
+ +Release 1.3 (2013-01-04) + +This is primarily a bug fix release. When this version is first +run on Linux, it removes any immutable bits from the Nix store and +increases the schema version of the Nix store. (The previous release +removed support for setting the immutable bit; this release clears any +remaining immutable bits to make certain operations more +efficient.) + +This release has contributions from Eelco Dolstra and Stuart +Pernsteiner. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.4.xml b/third_party/nix/doc/manual/release-notes/rl-1.4.xml new file mode 100644 index 000000000..aefb22f2b --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.4.xml @@ -0,0 +1,39 @@ +
+ +Release 1.4 (2013-02-26) + +This release fixes a security bug in multi-user operation. It +was possible for derivations to cause the mode of files outside of the +Nix store to be changed to 444 (read-only but world-readable) by +creating hard links to those files (details). + +There are also the following improvements: + + + + New built-in function: + builtins.hashString. + + Build logs are now stored in + /nix/var/log/nix/drvs/XX/, + where XX is the first two characters of + the derivation. This is useful on machines that keep a lot of build + logs (such as Hydra servers). + + The function corepkgs/fetchurl + can now make the downloaded file executable. This will allow + getting rid of all bootstrap binaries in the Nixpkgs source + tree. + + Language change: The expression "${./path} + ..." now evaluates to a string instead of a + path. + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.5.1.xml b/third_party/nix/doc/manual/release-notes/rl-1.5.1.xml new file mode 100644 index 000000000..035c8dbcb --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.5.1.xml @@ -0,0 +1,12 @@ +
+ +Release 1.5.1 (2013-02-28) + +The bug fix to the bug fix had a bug itself, of course. But +this time it will work for sure! + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.5.2.xml b/third_party/nix/doc/manual/release-notes/rl-1.5.2.xml new file mode 100644 index 000000000..7e81dd243 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.5.2.xml @@ -0,0 +1,12 @@ +
+ +Release 1.5.2 (2013-05-13) + +This is primarily a bug fix release. It has contributions from +Eelco Dolstra, Lluís Batlle i Rossell and Shea Levy. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.5.xml b/third_party/nix/doc/manual/release-notes/rl-1.5.xml new file mode 100644 index 000000000..8e279d769 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.5.xml @@ -0,0 +1,12 @@ +
+ +Release 1.5 (2013-02-27) + +This is a brown paper bag release to fix a regression introduced +by the hard link security fix in 1.4. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.6.1.xml b/third_party/nix/doc/manual/release-notes/rl-1.6.1.xml new file mode 100644 index 000000000..9ecc52734 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.6.1.xml @@ -0,0 +1,69 @@ +
+ +Release 1.6.1 (2013-10-28) + +This is primarily a bug fix release. Changes of interest +are: + + + + + Nix 1.6 accidentally changed the semantics of antiquoted + paths in strings, such as "${/foo}/bar". This + release reverts to the Nix 1.5.3 behaviour. + + + + Previously, Nix optimised expressions such as + "${expr}" to + expr. Thus it neither checked whether + expr could be coerced to a string, nor + applied such coercions. This meant that + "${123}" evaluatued to 123, + and "${./foo}" evaluated to + ./foo (even though + "${./foo} " evaluates to + "/nix/store/hash-foo "). + Nix now checks the type of antiquoted expressions and + applies coercions. + + + + Nix now shows the exact position of undefined variables. In + particular, undefined variable errors in a with + previously didn't show any position + information, so this makes it a lot easier to fix such + errors. + + + + Undefined variables are now treated consistently. + Previously, the tryEval function would catch + undefined variables inside a with but not + outside. Now tryEval never catches undefined + variables. + + + + Bash completion in nix-shell now works + correctly. + + + + Stack traces are less verbose: they no longer show calls to + builtin functions and only show a single line for each derivation + on the call stack. + + + + New built-in function: builtins.typeOf, + which returns the type of its argument as a string. + + + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.6.xml b/third_party/nix/doc/manual/release-notes/rl-1.6.xml new file mode 100644 index 000000000..580563420 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.6.xml @@ -0,0 +1,127 @@ +
+ +Release 1.6 (2013-09-10) + +In addition to the usual bug fixes, this release has several new +features: + + + + + The command nix-build --run-env has been + renamed to nix-shell. + + + + nix-shell now sources + $stdenv/setup inside the + interactive shell, rather than in a parent shell. This ensures + that shell functions defined by stdenv can be + used in the interactive shell. + + + + nix-shell has a new flag + to clear the environment, so you get an + environment that more closely corresponds to the “real” Nix build. + + + + + nix-shell now sets the shell prompt + (PS1) to ensure that Nix shells are distinguishable + from your regular shells. + + + + nix-env no longer requires a + * argument to match all packages, so + nix-env -qa is equivalent to nix-env + -qa '*'. + + + + nix-env -i has a new flag + () to remove all + previous packages from the profile. This makes it easier to do + declarative package management similar to NixOS’s + . For instance, if you + have a specification my-packages.nix like this: + + +with import <nixpkgs> {}; +[ thunderbird + geeqie + ... +] + + + then after any change to this file, you can run: + + +$ nix-env -f my-packages.nix -ir + + + to update your profile to match the specification. + + + + The ‘with’ language construct is now more + lazy. It only evaluates its argument if a variable might actually + refer to an attribute in the argument. For instance, this now + works: + + +let + pkgs = with pkgs; { foo = "old"; bar = foo; } // overrides; + overrides = { foo = "new"; }; +in pkgs.bar + + + This evaluates to "new", while previously it + gave an “infinite recursion” error. + + + + Nix now has proper integer arithmetic operators. For + instance, you can write x + y instead of + builtins.add x y, or x < + y instead of builtins.lessThan x y. + The comparison operators also work on strings. + + + + On 64-bit systems, Nix integers are now 64 bits rather than + 32 bits. + + + + When using the Nix daemon, the nix-daemon + worker process now runs on the same CPU as the client, on systems + that support setting CPU affinity. This gives a significant speedup + on some systems. + + + + If a stack overflow occurs in the Nix evaluator, you now get + a proper error message (rather than “Segmentation fault”) on some + systems. + + + + In addition to directories, you can now bind-mount regular + files in chroots through the (now misnamed) option + . + + + + +This release has contributions from Domen Kožar, Eelco Dolstra, +Florian Friesdorf, Gergely Risko, Ivan Kozik, Ludovic Courtès and Shea +Levy. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.7.xml b/third_party/nix/doc/manual/release-notes/rl-1.7.xml new file mode 100644 index 000000000..44ecaa78d --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.7.xml @@ -0,0 +1,263 @@ +
+ +Release 1.7 (2014-04-11) + +In addition to the usual bug fixes, this release has the +following new features: + + + + + Antiquotation is now allowed inside of quoted attribute + names (e.g. set."${foo}"). In the case where + the attribute name is just a single antiquotation, the quotes can + be dropped (e.g. the above example can be written + set.${foo}). If an attribute name inside of a + set declaration evaluates to null (e.g. + { ${null} = false; }), then that attribute is + not added to the set. + + + + Experimental support for cryptographically signed binary + caches. See the + commit for details. + + + + An experimental new substituter, + download-via-ssh, that fetches binaries from + remote machines via SSH. Specifying the flags --option + use-ssh-substituter true --option ssh-substituter-hosts + user@hostname will cause Nix + to download binaries from the specified machine, if it has + them. + + + + nix-store -r and + nix-build have a new flag, + , that builds a previously built + derivation again, and prints an error message if the output is not + exactly the same. This helps to verify whether a derivation is + truly deterministic. For example: + + +$ nix-build '<nixpkgs>' -A patchelf +… +$ nix-build '<nixpkgs>' -A patchelf --check +… +error: derivation `/nix/store/1ipvxs…-patchelf-0.6' may not be deterministic: + hash mismatch in output `/nix/store/4pc1dm…-patchelf-0.6.drv' + + + + + + + + The nix-instantiate flags + and + have been renamed to and + , respectively. + + + + nix-instantiate, + nix-build and nix-shell now + have a flag (or ) that + allows you to specify the expression to be evaluated as a command + line argument. For instance, nix-instantiate --eval -E + '1 + 2' will print 3. + + + + nix-shell improvements: + + + + + It has a new flag, (or + ), that sets up a build environment + containing the specified packages from Nixpkgs. For example, + the command + + +$ nix-shell -p sqlite xorg.libX11 hello + + + will start a shell in which the given packages are + present. + + + + It now uses shell.nix as the + default expression, falling back to + default.nix if the former doesn’t + exist. This makes it convenient to have a + shell.nix in your project to set up a + nice development environment. + + + + It evaluates the derivation attribute + shellHook, if set. Since + stdenv does not normally execute this hook, + it allows you to do nix-shell-specific + setup. + + + + It preserves the user’s timezone setting. + + + + + + + + In chroots, Nix now sets up a /dev + containing only a minimal set of devices (such as + /dev/null). Note that it only does this if + you don’t have /dev + listed in your setting; + otherwise, it will bind-mount the /dev from + outside the chroot. + + Similarly, if you don’t have /dev/pts listed + in , Nix will mount a private + devpts filesystem on the chroot’s + /dev/pts. + + + + + New built-in function: builtins.toJSON, + which returns a JSON representation of a value. + + + + nix-env -q has a new flag + to print a JSON representation of the + installed or available packages. + + + + nix-env now supports meta attributes with + more complex values, such as attribute sets. + + + + The flag now allows attribute names with + dots in them, e.g. + + +$ nix-instantiate --eval '<nixos>' -A 'config.systemd.units."nscd.service".text' + + + + + + + The option to + nix-store --gc now accepts a unit + specifier. For example, nix-store --gc --max-freed + 1G will free up to 1 gigabyte of disk space. + + + + nix-collect-garbage has a new flag + + Nd, which deletes + all user environment generations older than + N days. Likewise, nix-env + --delete-generations accepts a + Nd age limit. + + + + Nix now heuristically detects whether a build failure was + due to a disk-full condition. In that case, the build is not + flagged as “permanently failed”. This is mostly useful for Hydra, + which needs to distinguish between permanent and transient build + failures. + + + + There is a new symbol __curPos that + expands to an attribute set containing its file name and line and + column numbers, e.g. { file = "foo.nix"; line = 10; + column = 5; }. There also is a new builtin function, + unsafeGetAttrPos, that returns the position of + an attribute. This is used by Nixpkgs to provide location + information in error messages, e.g. + + +$ nix-build '<nixpkgs>' -A libreoffice --argstr system x86_64-darwin +error: the package ‘libreoffice-4.0.5.2’ in ‘.../applications/office/libreoffice/default.nix:263’ + is not supported on ‘x86_64-darwin’ + + + + + + + The garbage collector is now more concurrent with other Nix + processes because it releases certain locks earlier. + + + + The binary tarball installer has been improved. You can now + install Nix by running: + + +$ bash <(curl https://nixos.org/nix/install) + + + + + + + More evaluation errors include position information. For + instance, selecting a missing attribute will print something like + + +error: attribute `nixUnstabl' missing, at /etc/nixos/configurations/misc/eelco/mandark.nix:216:15 + + + + + + + The command nix-setuid-helper is + gone. + + + + Nix no longer uses Automake, but instead has a + non-recursive, GNU Make-based build system. + + + + All installed libraries now have the prefix + libnix. In particular, this gets rid of + libutil, which could clash with libraries with + the same name from other packages. + + + + Nix now requires a compiler that supports C++11. + + + + +This release has contributions from Danny Wilson, Domen Kožar, +Eelco Dolstra, Ian-Woo Kim, Ludovic Courtès, Maxim Ivanov, Petr +Rockai, Ricardo M. Correia and Shea Levy. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.8.xml b/third_party/nix/doc/manual/release-notes/rl-1.8.xml new file mode 100644 index 000000000..c854c5c5f --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.8.xml @@ -0,0 +1,123 @@ +
+ +Release 1.8 (2014-12-14) + + + + Breaking change: to address a race condition, the + remote build hook mechanism now uses nix-store + --serve on the remote machine. This requires build slaves + to be updated to Nix 1.8. + + Nix now uses HTTPS instead of HTTP to access the + default binary cache, + cache.nixos.org. + + nix-env selectors are now regular + expressions. For instance, you can do + + +$ nix-env -qa '.*zip.*' + + + to query all packages with a name containing + zip. + + nix-store --read-log can now + fetch remote build logs. If a build log is not available locally, + then ‘nix-store -l’ will now try to download it from the servers + listed in the ‘log-servers’ option in nix.conf. For instance, if you + have the configuration option + + +log-servers = http://hydra.nixos.org/log + + +then it will try to get logs from +http://hydra.nixos.org/log/base name of the +store path. This allows you to do things like: + + +$ nix-store -l $(which xterm) + + + and get a log even if xterm wasn't built + locally. + + New builtin functions: + attrValues, deepSeq, + fromJSON, readDir, + seq. + + nix-instantiate --eval now has a + flag to print the resulting value in JSON + format. + + nix-copy-closure now uses + nix-store --serve on the remote side to send or + receive closures. This fixes a race condition between + nix-copy-closure and the garbage + collector. + + Derivations can specify the new special attribute + allowedRequisites, which has a similar meaning to + allowedReferences. But instead of only enforcing + to explicitly specify the immediate references, it requires the + derivation to specify all the dependencies recursively (hence the + name, requisites) that are used by the resulting + output. + + On Mac OS X, Nix now handles case collisions when + importing closures from case-sensitive file systems. This is mostly + useful for running NixOps on Mac OS X. + + The Nix daemon has new configuration options + (specifying the users and groups that + are allowed to connect to the daemon) and + (specifying the users and groups that + can perform privileged operations like specifying untrusted binary + caches). + + The configuration option + now defaults to the number of available + CPU cores. + + Build users are now used by default when Nix is + invoked as root. This prevents builds from accidentally running as + root. + + Nix now includes systemd units and Upstart + jobs. + + Speed improvements to nix-store + --optimise. + + Language change: the == operator + now ignores string contexts (the “dependencies” of a + string). + + Nix now filters out Nix-specific ANSI escape + sequences on standard error. They are supposed to be invisible, but + some terminals show them anyway. + + Various commands now automatically pipe their output + into the pager as specified by the PAGER environment + variable. + + Several improvements to reduce memory consumption in + the evaluator. + + + +This release has contributions from Adam Szkoda, Aristid +Breitkreuz, Bob van der Linden, Charles Strahan, darealshinji, Eelco +Dolstra, Gergely Risko, Joel Taylor, Ludovic Courtès, Marko Durkovic, +Mikey Ariel, Paul Colomiets, Ricardo M. Correia, Ricky Elrod, Robert +Helgesson, Rob Vermaas, Russell O'Connor, Shea Levy, Shell Turner, +Sönke Hahn, Steve Purcell, Vladimír Čunát and Wout Mertens. + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-1.9.xml b/third_party/nix/doc/manual/release-notes/rl-1.9.xml new file mode 100644 index 000000000..c8406bd20 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-1.9.xml @@ -0,0 +1,216 @@ +
+ +Release 1.9 (2015-06-12) + +In addition to the usual bug fixes, this release has the +following new features: + + + + + Signed binary cache support. You can enable signature + checking by adding the following to nix.conf: + + +signed-binary-caches = * +binary-cache-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= + + + This will prevent Nix from downloading any binary from the cache + that is not signed by one of the keys listed in + . + + Signature checking is only supported if you built Nix with + the libsodium package. + + Note that while Nix has had experimental support for signed + binary caches since version 1.7, this release changes the + signature format in a backwards-incompatible way. + + + + + + Automatic downloading of Nix expression tarballs. In various + places, you can now specify the URL of a tarball containing Nix + expressions (such as Nixpkgs), which will be downloaded and + unpacked automatically. For example: + + + + In nix-env: + + +$ nix-env -f https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz -iA firefox + + + This installs Firefox from the latest tested and built revision + of the NixOS 14.12 channel. + + In nix-build and + nix-shell: + + +$ nix-build https://github.com/NixOS/nixpkgs/archive/master.tar.gz -A hello + + + This builds GNU Hello from the latest revision of the Nixpkgs + master branch. + + In the Nix search path (as specified via + NIX_PATH or ). For example, to + start a shell containing the Pan package from a specific version + of Nixpkgs: + + +$ nix-shell -p pan -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/8a3eea054838b55aca962c3fbde9c83c102b8bf2.tar.gz + + + + + In nixos-rebuild (on NixOS): + + +$ nixos-rebuild test -I nixpkgs=https://github.com/NixOS/nixpkgs-channels/archive/nixos-unstable.tar.gz + + + + + In Nix expressions, via the new builtin function fetchTarball: + + +with import (fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-14.12.tar.gz) {}; … + + + (This is not allowed in restricted mode.) + + + + + + + + nix-shell improvements: + + + + nix-shell now has a flag + to execute a command in the + nix-shell environment, + e.g. nix-shell --run make. This is like + the existing flag, except that it + uses a non-interactive shell (ensuring that hitting Ctrl-C won’t + drop you into the child shell). + + nix-shell can now be used as + a #!-interpreter. This allows you to write + scripts that dynamically fetch their own dependencies. For + example, here is a Haskell script that, when invoked, first + downloads GHC and the Haskell packages on which it depends: + + +#! /usr/bin/env nix-shell +#! nix-shell -i runghc -p haskellPackages.ghc haskellPackages.HTTP + +import Network.HTTP + +main = do + resp <- Network.HTTP.simpleHTTP (getRequest "http://nixos.org/") + body <- getResponseBody resp + print (take 100 body) + + + Of course, the dependencies are cached in the Nix store, so the + second invocation of this script will be much + faster. + + + + + + + + Chroot improvements: + + + + Chroot builds are now supported on Mac OS X + (using its sandbox mechanism). + + If chroots are enabled, they are now used for + all derivations, including fixed-output derivations (such as + fetchurl). The latter do have network + access, but can no longer access the host filesystem. If you + need the old behaviour, you can set the option + to + relaxed. + + On Linux, if chroots are enabled, builds are + performed in a private PID namespace once again. (This + functionality was lost in Nix 1.8.) + + Store paths listed in + are now automatically + expanded to their closure. For instance, if you want + /nix/store/…-bash/bin/sh mounted in your + chroot as /bin/sh, you only need to say + build-chroot-dirs = + /bin/sh=/nix/store/…-bash/bin/sh; it is no longer + necessary to specify the dependencies of Bash. + + + + + + The new derivation attribute + passAsFile allows you to specify that the + contents of derivation attributes should be passed via files rather + than environment variables. This is useful if you need to pass very + long strings that exceed the size limit of the environment. The + Nixpkgs function writeTextFile uses + this. + + You can now use ~ in Nix file + names to refer to your home directory, e.g. import + ~/.nixpkgs/config.nix. + + Nix has a new option + that allows limiting what paths the Nix evaluator has access to. By + passing --option restrict-eval true to Nix, the + evaluator will throw an exception if an attempt is made to access + any file outside of the Nix search path. This is primarily intended + for Hydra to ensure that a Hydra jobset only refers to its declared + inputs (and is therefore reproducible). + + nix-env now only creates a new + “generation” symlink in /nix/var/nix/profiles + if something actually changed. + + The environment variable NIX_PAGER + can now be set to override PAGER. You can set it to + cat to disable paging for Nix commands + only. + + Failing <...> + lookups now show position information. + + Improved Boehm GC use: we disabled scanning for + interior pointers, which should reduce the “Repeated + allocation of very large block” warnings and associated + retention of memory. + + + +This release has contributions from aszlig, Benjamin Staffin, +Charles Strahan, Christian Theune, Daniel Hahler, Danylo Hlynskyi +Daniel Peebles, Dan Peebles, Domen Kožar, Eelco Dolstra, Harald van +Dijk, Hoang Xuan Phu, Jaka Hudoklin, Jeff Ramnani, j-keck, Linquize, +Luca Bruno, Michael Merickel, Oliver Dunkl, Rob Vermaas, Rok Garbas, +Shea Levy, Tobias Geerinckx-Rice and William A. Kennington III. + +
+ diff --git a/third_party/nix/doc/manual/release-notes/rl-2.0.xml b/third_party/nix/doc/manual/release-notes/rl-2.0.xml new file mode 100644 index 000000000..fc9a77b08 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.0.xml @@ -0,0 +1,1012 @@ +
+ +Release 2.0 (2018-02-22) + +The following incompatible changes have been made: + + + + + The manifest-based substituter mechanism + (download-using-manifests) has been removed. It + has been superseded by the binary cache substituter mechanism + since several years. As a result, the following programs have been + removed: + + + nix-pull + nix-generate-patches + bsdiff + bspatch + + + + + + The “copy from other stores” substituter mechanism + (copy-from-other-stores and the + NIX_OTHER_STORES environment variable) has been + removed. It was primarily used by the NixOS installer to copy + available paths from the installation medium. The replacement is + to use a chroot store as a substituter + (e.g. --substituters /mnt), or to build into a + chroot store (e.g. --store /mnt --substituters /). + + + + The command nix-push has been removed as + part of the effort to eliminate Nix's dependency on Perl. You can + use nix copy instead, e.g. nix copy + --to file:///tmp/my-binary-cache paths… + + + + The “nested” log output feature () has been removed. As a result, + nix-log2xml was also removed. + + + + OpenSSL-based signing has been removed. This + feature was never well-supported. A better alternative is provided + by the and + options. + + + + Failed build caching has been removed. This + feature was introduced to support the Hydra continuous build + system, but Hydra no longer uses it. + + + + nix-mode.el has been removed from + Nix. It is now a separate + repository and can be installed through the MELPA package + repository. + + + + +This release has the following new features: + + + + + It introduces a new command named nix, + which is intended to eventually replace all + nix-* commands with a more consistent and + better designed user interface. It currently provides replacements + for some (but not all) of the functionality provided by + nix-store, nix-build, + nix-shell -p, nix-env -qa, + nix-instantiate --eval, + nix-push and + nix-copy-closure. It has the following major + features: + + + + + Unlike the legacy commands, it has a consistent way to + refer to packages and package-like arguments (like store + paths). For example, the following commands all copy the GNU + Hello package to a remote machine: + + nix copy --to ssh://machine nixpkgs.hello + nix copy --to ssh://machine /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + nix copy --to ssh://machine '(with import <nixpkgs> {}; hello)' + + By contrast, nix-copy-closure only accepted + store paths as arguments. + + + + It is self-documenting: shows + all available command-line arguments. If + is given after a subcommand, it shows + examples for that subcommand. nix + --help-config shows all configuration + options. + + + + It is much less verbose. By default, it displays a + single-line progress indicator that shows how many packages + are left to be built or downloaded, and (if there are running + builds) the most recent line of builder output. If a build + fails, it shows the last few lines of builder output. The full + build log can be retrieved using nix + log. + + + + It provides + all nix.conf configuration options as + command line flags. For example, instead of --option + http-connections 100 you can write + --http-connections 100. Boolean options can + be written as + --foo or + --no-foo + (e.g. ). + + + + Many subcommands have a flag to + write results to stdout in JSON format. + + + + + Please note that the nix command + is a work in progress and the interface is subject to + change. + + It provides the following high-level (“porcelain”) + subcommands: + + + + + nix build is a replacement for + nix-build. + + + + nix run executes a command in an + environment in which the specified packages are available. It + is (roughly) a replacement for nix-shell + -p. Unlike that command, it does not execute the + command in a shell, and has a flag (-c) + that specifies the unquoted command line to be + executed. + + It is particularly useful in conjunction with chroot + stores, allowing Linux users who do not have permission to + install Nix in /nix/store to still use + binary substitutes that assume + /nix/store. For example, + + nix run --store ~/my-nix nixpkgs.hello -c hello --greeting 'Hi everybody!' + + downloads (or if not substitutes are available, builds) the + GNU Hello package into + ~/my-nix/nix/store, then runs + hello in a mount namespace where + ~/my-nix/nix/store is mounted onto + /nix/store. + + + + nix search replaces nix-env + -qa. It searches the available packages for + occurrences of a search string in the attribute name, package + name or description. Unlike nix-env -qa, it + has a cache to speed up subsequent searches. + + + + nix copy copies paths between + arbitrary Nix stores, generalising + nix-copy-closure and + nix-push. + + + + nix repl replaces the external + program nix-repl. It provides an + interactive environment for evaluating and building Nix + expressions. Note that it uses linenoise-ng + instead of GNU Readline. + + + + nix upgrade-nix upgrades Nix to the + latest stable version. This requires that Nix is installed in + a profile. (Thus it won’t work on NixOS, or if it’s installed + outside of the Nix store.) + + + + nix verify checks whether store paths + are unmodified and/or “trusted” (see below). It replaces + nix-store --verify and nix-store + --verify-path. + + + + nix log shows the build log of a + package or path. If the build log is not available locally, it + will try to obtain it from the configured substituters (such + as cache.nixos.org, which now provides build + logs). + + + + nix edit opens the source code of a + package in your editor. + + + + nix eval replaces + nix-instantiate --eval. + + + + nix + why-depends shows why one store path has another in + its closure. This is primarily useful to finding the causes of + closure bloat. For example, + + nix why-depends nixpkgs.vlc nixpkgs.libdrm.dev + + shows a chain of files and fragments of file contents that + cause the VLC package to have the “dev” output of + libdrm in its closure — an undesirable + situation. + + + + nix path-info shows information about + store paths, replacing nix-store -q. A + useful feature is the option + (). For example, the following command show + the closure sizes of every path in the current NixOS system + closure, sorted by size: + + nix path-info -rS /run/current-system | sort -nk2 + + + + + + nix optimise-store replaces + nix-store --optimise. The main difference + is that it has a progress indicator. + + + + + A number of low-level (“plumbing”) commands are also + available: + + + + + nix ls-store and nix + ls-nar list the contents of a store path or NAR + file. The former is primarily useful in conjunction with + remote stores, e.g. + + nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + + lists the contents of path in a binary cache. + + + + nix cat-store and nix + cat-nar allow extracting a file from a store path or + NAR file. + + + + nix dump-path writes the contents of + a store path to stdout in NAR format. This replaces + nix-store --dump. + + + + nix + show-derivation displays a store derivation in JSON + format. This is an alternative to + pp-aterm. + + + + nix + add-to-store replaces nix-store + --add. + + + + nix sign-paths signs store + paths. + + + + nix copy-sigs copies signatures from + one store to another. + + + + nix show-config shows all + configuration options and their current values. + + + + + + + + The store abstraction that Nix has had for a long time to + support store access via the Nix daemon has been extended + significantly. In particular, substituters (which used to be + external programs such as + download-from-binary-cache) are now subclasses + of the abstract Store class. This allows + many Nix commands to operate on such store types. For example, + nix path-info shows information about paths in + your local Nix store, while nix path-info --store + https://cache.nixos.org/ shows information about paths + in the specified binary cache. Similarly, + nix-copy-closure, nix-push + and substitution are all instances of the general notion of + copying paths between different kinds of Nix stores. + + Stores are specified using an URI-like syntax, + e.g. https://cache.nixos.org/ or + ssh://machine. The following store types are supported: + + + + + + LocalStore (stori URI + local or an absolute path) and the misnamed + RemoteStore (daemon) + provide access to a local Nix store, the latter via the Nix + daemon. You can use auto or the empty + string to auto-select a local or daemon store depending on + whether you have write permission to the Nix store. It is no + longer necessary to set the NIX_REMOTE + environment variable to use the Nix daemon. + + As noted above, LocalStore now + supports chroot builds, allowing the “physical” location of + the Nix store + (e.g. /home/alice/nix/store) to differ + from its “logical” location (typically + /nix/store). This allows non-root users + to use Nix while still getting the benefits from prebuilt + binaries from cache.nixos.org. + + + + + + BinaryCacheStore is the abstract + superclass of all binary cache stores. It supports writing + build logs and NAR content listings in JSON format. + + + + + + HttpBinaryCacheStore + (http://, https://) + supports binary caches via HTTP or HTTPS. If the server + supports PUT requests, it supports + uploading store paths via commands such as nix + copy. + + + + + + LocalBinaryCacheStore + (file://) supports binary caches in the + local filesystem. + + + + + + S3BinaryCacheStore + (s3://) supports binary caches stored in + Amazon S3, if enabled at compile time. + + + + + + LegacySSHStore (ssh://) + is used to implement remote builds and + nix-copy-closure. + + + + + + SSHStore + (ssh-ng://) supports arbitrary Nix + operations on a remote machine via the same protocol used by + nix-daemon. + + + + + + + + + + + + Security has been improved in various ways: + + + + + Nix now stores signatures for local store + paths. When paths are copied between stores (e.g., copied from + a binary cache to a local store), signatures are + propagated. + + Locally-built paths are signed automatically using the + secret keys specified by the + store option. Secret/public key pairs can be generated using + nix-store + --generate-binary-cache-key. + + In addition, locally-built store paths are marked as + “ultimately trusted”, but this bit is not propagated when + paths are copied between stores. + + + + Content-addressable store paths no longer require + signatures — they can be imported into a store by unprivileged + users even if they lack signatures. + + + + The command nix verify checks whether + the specified paths are trusted, i.e., have a certain number + of trusted signatures, are ultimately trusted, or are + content-addressed. + + + + Substitutions from binary caches now + require signatures by default. This was already the case on + NixOS. + + + + In Linux sandbox builds, we now + use /build instead of + /tmp as the temporary build + directory. This fixes potential security problems when a build + accidentally stores its TMPDIR in some + security-sensitive place, such as an RPATH. + + + + + + + + + + Pure evaluation mode. This is a variant + of the existing restricted evaluation mode. In pure mode, the Nix + evaluator forbids access to anything that could cause different + evaluations of the same command line arguments to produce a + different result. This includes builtin functions such as + builtins.getEnv, but more importantly, + all filesystem or network access unless a + content hash or commit hash is specified. For example, calls to + builtins.fetchGit are only allowed if a + rev attribute is specified. + + The goal of this feature is to enable true reproducibility + and traceability of builds (including NixOS system configurations) + at the evaluation level. For example, in the future, + nixos-rebuild might build configurations from a + Nix expression in a Git repository in pure mode. That expression + might fetch other repositories such as Nixpkgs via + builtins.fetchGit. The commit hash of the + top-level repository then uniquely identifies a running system, + and, in conjunction with that repository, allows it to be + reproduced or modified. + + + + + There are several new features to support binary + reproducibility (i.e. to help ensure that multiple builds of the + same derivation produce exactly the same output). When + is set to + false, it’s no + longer a fatal error if build rounds produce different + output. Also, a hook named is provided + to allow you to run tools such as diffoscope + when build rounds produce different output. + + + + Configuring remote builds is a lot easier now. Provided you + are not using the Nix daemon, you can now just specify a remote + build machine on the command line, e.g. --option builders + 'ssh://my-mac x86_64-darwin'. The environment variable + NIX_BUILD_HOOK has been removed and is no longer + needed. The environment variable NIX_REMOTE_SYSTEMS + is still supported for compatibility, but it is also possible to + specify builders in nix.conf by setting the + option builders = + @path. + + + + If a fixed-output derivation produces a result with an + incorrect hash, the output path is moved to the location + corresponding to the actual hash and registered as valid. Thus, a + subsequent build of the fixed-output derivation with the correct + hash is unnecessary. + + + + nix-shell now + sets the IN_NIX_SHELL environment variable + during evaluation and in the shell itself. This can be used to + perform different actions depending on whether you’re in a Nix + shell or in a regular build. Nixpkgs provides + lib.inNixShell to check this variable during + evaluation. + + + + NIX_PATH is now lazy, so URIs in the path are + only downloaded if they are needed for evaluation. + + + + You can now use + channel:channel-name as a + short-hand for + https://nixos.org/channels/channel-name/nixexprs.tar.xz. For + example, nix-build channel:nixos-15.09 -A hello + will build the GNU Hello package from the + nixos-15.09 channel. In the future, this may + use Git to fetch updates more efficiently. + + + + When is given, the last + 10 lines of the build log will be shown if a build + fails. + + + + Networking has been improved: + + + + + HTTP/2 is now supported. This makes binary cache lookups + much + more efficient. + + + + We now retry downloads on many HTTP errors, making + binary caches substituters more resilient to temporary + failures. + + + + HTTP credentials can now be configured via the standard + netrc mechanism. + + + + If S3 support is enabled at compile time, + s3:// URIs are supported + in all places where Nix allows URIs. + + + + Brotli compression is now supported. In particular, + cache.nixos.org build logs are now compressed using + Brotli. + + + + + + + + + + nix-env now + ignores packages with bad derivation names (in particular those + starting with a digit or containing a dot). + + + + Many configuration options have been renamed, either because + they were unnecessarily verbose + (e.g. is now just + ) or to reflect generalised behaviour + (e.g. is now + because it allows arbitrary store + URIs). The old names are still supported for compatibility. + + + + The option can now + be set to auto to use the number of CPUs in the + system. + + + + Hashes can now + be specified in base-64 format, in addition to base-16 and the + non-standard base-32. + + + + nix-shell now uses + bashInteractive from Nixpkgs, rather than the + bash command that happens to be in the caller’s + PATH. This is especially important on macOS where + the bash provided by the system is seriously + outdated and cannot execute stdenv’s setup + script. + + + + Nix can now automatically trigger a garbage collection if + free disk space drops below a certain level during a build. This + is configured using the and + options. + + + + nix-store -q --roots and + nix-store --gc --print-roots now show temporary + and in-memory roots. + + + + + Nix can now be extended with plugins. See the documentation of + the option for more details. + + + + + +The Nix language has the following new features: + + + + + It supports floating point numbers. They are based on the + C++ float type and are supported by the + existing numerical operators. Export and import to and from JSON + and XML works, too. + + + + Derivation attributes can now reference the outputs of the + derivation using the placeholder builtin + function. For example, the attribute + + +configureFlags = "--prefix=${placeholder "out"} --includedir=${placeholder "dev"}"; + + + will cause the configureFlags environment variable + to contain the actual store paths corresponding to the + out and dev outputs. + + + + + + +The following builtin functions are new or extended: + + + + + builtins.fetchGit + allows Git repositories to be fetched at evaluation time. Thus it + differs from the fetchgit function in + Nixpkgs, which fetches at build time and cannot be used to fetch + Nix expressions during evaluation. A typical use case is to import + external NixOS modules from your configuration, e.g. + + imports = [ (builtins.fetchGit https://github.com/edolstra/dwarffs + "/module.nix") ]; + + + + + + Similarly, builtins.fetchMercurial + allows you to fetch Mercurial repositories. + + + + builtins.path generalises + builtins.filterSource and path literals + (e.g. ./foo). It allows specifying a store path + name that differs from the source path name + (e.g. builtins.path { path = ./foo; name = "bar"; + }) and also supports filtering out unwanted + files. + + + + builtins.fetchurl and + builtins.fetchTarball now support + sha256 and name + attributes. + + + + builtins.split + splits a string using a POSIX extended regular expression as the + separator. + + + + builtins.partition + partitions the elements of a list into two lists, depending on a + Boolean predicate. + + + + <nix/fetchurl.nix> now uses the + content-addressable tarball cache at + http://tarballs.nixos.org/, just like + fetchurl in + Nixpkgs. (f2682e6e18a76ecbfb8a12c17e3a0ca15c084197) + + + + In restricted and pure evaluation mode, builtin functions + that download from the network (such as + fetchGit) are permitted to fetch underneath a + list of URI prefixes specified in the option + . + + + + + + +The Nix build environment has the following changes: + + + + + Values such as Booleans, integers, (nested) lists and + attribute sets can now + be passed to builders in a non-lossy way. If the special attribute + __structuredAttrs is set to + true, the other derivation attributes are + serialised in JSON format and made available to the builder via + the file .attrs.json in the builder’s temporary + directory. This obviates the need for + passAsFile since JSON files have no size + restrictions, unlike process environments. + + As + a convenience to Bash builders, Nix writes a script named + .attrs.sh to the builder’s directory that + initialises shell variables corresponding to all attributes that + are representable in Bash. This includes non-nested (associative) + arrays. For example, the attribute hardening.format = + true ends up as the Bash associative array element + ${hardening[format]}. + + + + Builders can now + communicate what build phase they are in by writing messages to + the file descriptor specified in NIX_LOG_FD. The + current phase is shown by the nix progress + indicator. + + + + + In Linux sandbox builds, we now + provide a default /bin/sh (namely + ash from BusyBox). + + + + In structured attribute mode, + exportReferencesGraph exports + extended information about closures in JSON format. In particular, + it includes the sizes and hashes of paths. This is primarily + useful for NixOS image builders. + + + + Builds are now + killed as soon as Nix receives EOF on the builder’s stdout or + stderr. This fixes a bug that allowed builds to hang Nix + indefinitely, regardless of + timeouts. + + + + The configuration + option can now specify optional paths by appending a + ?, e.g. /dev/nvidiactl? will + bind-mount /dev/nvidiactl only if it + exists. + + + + On Linux, builds are now executed in a user + namespace with UID 1000 and GID 100. + + + + + + +A number of significant internal changes were made: + + + + + Nix no longer depends on Perl and all Perl components have + been rewritten in C++ or removed. The Perl bindings that used to + be part of Nix have been moved to a separate package, + nix-perl. + + + + All Store classes are now + thread-safe. RemoteStore supports multiple + concurrent connections to the daemon. This is primarily useful in + multi-threaded programs such as + hydra-queue-runner. + + + + + + +This release has contributions from + +Adrien Devresse, +Alexander Ried, +Alex Cruice, +Alexey Shmalko, +AmineChikhaoui, +Andy Wingo, +Aneesh Agrawal, +Anthony Cowley, +Armijn Hemel, +aszlig, +Ben Gamari, +Benjamin Hipple, +Benjamin Staffin, +Benno Fünfstück, +Bjørn Forsman, +Brian McKenna, +Charles Strahan, +Chase Adams, +Chris Martin, +Christian Theune, +Chris Warburton, +Daiderd Jordan, +Dan Connolly, +Daniel Peebles, +Dan Peebles, +davidak, +David McFarland, +Dmitry Kalinkin, +Domen Kožar, +Eelco Dolstra, +Emery Hemingway, +Eric Litak, +Eric Wolf, +Fabian Schmitthenner, +Frederik Rietdijk, +Gabriel Gonzalez, +Giorgio Gallo, +Graham Christensen, +Guillaume Maudoux, +Harmen, +Iavael, +James Broadhead, +James Earl Douglas, +Janus Troelsen, +Jeremy Shaw, +Joachim Schiele, +Joe Hermaszewski, +Joel Moberg, +Johannes 'fish' Ziemke, +Jörg Thalheim, +Jude Taylor, +kballou, +Keshav Kini, +Kjetil Orbekk, +Langston Barrett, +Linus Heckemann, +Ludovic Courtès, +Manav Rathi, +Marc Scholten, +Markus Hauck, +Matt Audesse, +Matthew Bauer, +Matthias Beyer, +Matthieu Coudron, +N1X, +Nathan Zadoks, +Neil Mayhew, +Nicolas B. Pierron, +Niklas Hambüchen, +Nikolay Amiantov, +Ole Jørgen Brønner, +Orivej Desh, +Peter Simons, +Peter Stuart, +Pyry Jahkola, +regnat, +Renzo Carbonara, +Rhys, +Robert Vollmert, +Scott Olson, +Scott R. Parish, +Sergei Trofimovich, +Shea Levy, +Sheena Artrip, +Spencer Baugh, +Stefan Junker, +Susan Potter, +Thomas Tuegel, +Timothy Allen, +Tristan Hume, +Tuomas Tynkkynen, +tv, +Tyson Whitehead, +Vladimír Čunát, +Will Dietz, +wmertens, +Wout Mertens, +zimbatm and +Zoran Plesivčak. + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-2.1.xml b/third_party/nix/doc/manual/release-notes/rl-2.1.xml new file mode 100644 index 000000000..16c243fc1 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.1.xml @@ -0,0 +1,133 @@ +
+ +Release 2.1 (2018-09-02) + +This is primarily a bug fix release. It also reduces memory +consumption in certain situations. In addition, it has the following +new features: + + + + + The Nix installer will no longer default to the Multi-User + installation for macOS. You can still instruct the installer to + run in multi-user mode. + + + + + The Nix installer now supports performing a Multi-User + installation for Linux computers which are running systemd. You + can select a Multi-User installation by passing the + flag to the installer: sh <(curl + https://nixos.org/nix/install) --daemon. + + + The multi-user installer cannot handle systems with SELinux. + If your system has SELinux enabled, you can force the installer to run + in single-user mode. + + + + New builtin functions: + builtins.bitAnd, + builtins.bitOr, + builtins.bitXor, + builtins.fromTOML, + builtins.concatMap, + builtins.mapAttrs. + + + + + The S3 binary cache store now supports uploading NARs larger + than 5 GiB. + + + + The S3 binary cache store now supports uploading to + S3-compatible services with the endpoint + option. + + + + The flag is no longer required + to recover from disappeared NARs in binary caches. + + + + nix-daemon now respects + . + + + + nix run now respects + nix-support/propagated-user-env-packages. + + + + +This release has contributions from + +Adrien Devresse, +Aleksandr Pashkov, +Alexandre Esteves, +Amine Chikhaoui, +Andrew Dunham, +Asad Saeeduddin, +aszlig, +Ben Challenor, +Ben Gamari, +Benjamin Hipple, +Bogdan Seniuc, +Corey O'Connor, +Daiderd Jordan, +Daniel Peebles, +Daniel Poelzleithner, +Danylo Hlynskyi, +Dmitry Kalinkin, +Domen Kožar, +Doug Beardsley, +Eelco Dolstra, +Erik Arvstedt, +Félix Baylac-Jacqué, +Gleb Peregud, +Graham Christensen, +Guillaume Maudoux, +Ivan Kozik, +John Arnold, +Justin Humm, +Linus Heckemann, +Lorenzo Manacorda, +Matthew Justin Bauer, +Matthew O'Gorman, +Maximilian Bosch, +Michael Bishop, +Michael Fiano, +Michael Mercier, +Michael Raskin, +Michael Weiss, +Nicolas Dudebout, +Peter Simons, +Ryan Trinkle, +Samuel Dionne-Riel, +Sean Seefried, +Shea Levy, +Symphorien Gibol, +Tim Engler, +Tim Sears, +Tuomas Tynkkynen, +volth, +Will Dietz, +Yorick van Pelt and +zimbatm. + + +
diff --git a/third_party/nix/doc/manual/release-notes/rl-2.2.xml b/third_party/nix/doc/manual/release-notes/rl-2.2.xml new file mode 100644 index 000000000..d29eb87e8 --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.2.xml @@ -0,0 +1,143 @@ +
+ +Release 2.2 (2019-01-11) + +This is primarily a bug fix release. It also has the following +changes: + + + + + In derivations that use structured attributes (i.e. that + specify set the __structuredAttrs attribute to + true to cause all attributes to be passed to + the builder in JSON format), you can now specify closure checks + per output, e.g.: + + +outputChecks."out" = { + # The closure of 'out' must not be larger than 256 MiB. + maxClosureSize = 256 * 1024 * 1024; + + # It must not refer to C compiler or to the 'dev' output. + disallowedRequisites = [ stdenv.cc "dev" ]; +}; + +outputChecks."dev" = { + # The 'dev' output must not be larger than 128 KiB. + maxSize = 128 * 1024; +}; + + + + + + + + The derivation attribute + requiredSystemFeatures is now enforced for + local builds, and not just to route builds to remote builders. + The supported features of a machine can be specified through the + configuration setting system-features. + + By default, system-features includes + kvm if /dev/kvm + exists. For compatibility, it also includes the pseudo-features + nixos-test, benchmark and + big-parallel which are used by Nixpkgs to route + builds to particular Hydra build machines. + + + + + Sandbox builds are now enabled by default on Linux. + + + + The new command nix doctor shows + potential issues with your Nix installation. + + + + The fetchGit builtin function now uses a + caching scheme that puts different remote repositories in distinct + local repositories, rather than a single shared repository. This + may require more disk space but is faster. + + + + The dirOf builtin function now works on + relative paths. + + + + Nix now supports SRI hashes, + allowing the hash algorithm and hash to be specified in a single + string. For example, you can write: + + +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + hash = "sha256-XSLa0FjVyADWWhFfkZ2iKTjFDda6mMXjoYMXLRSYQKQ="; +}; + + + instead of + + +import <nix/fetchurl.nix> { + url = https://nixos.org/releases/nix/nix-2.1.3/nix-2.1.3.tar.xz; + sha256 = "5d22dad058d5c800d65a115f919da22938c50dd6ba98c5e3a183172d149840a4"; +}; + + + + + In fixed-output derivations, the + outputHashAlgo attribute is no longer mandatory + if outputHash specifies the hash. + + nix hash-file and nix + hash-path now print hashes in SRI format by + default. They also use SHA-256 by default instead of SHA-512 + because that's what we use most of the time in Nixpkgs. + + + + Integers are now 64 bits on all platforms. + + + + The evaluator now prints profiling statistics (enabled via + the NIX_SHOW_STATS and + NIX_COUNT_CALLS environment variables) in JSON + format. + + + + The option in nix-store + --query has been removed. Instead, there now is an + option to output the dependency graph + in GraphML format. + + + + All nix-* commands are now symlinks to + nix. This saves a bit of disk space. + + + + nix repl now uses + libeditline or + libreadline. + + + + +
+ diff --git a/third_party/nix/doc/manual/release-notes/rl-2.3.xml b/third_party/nix/doc/manual/release-notes/rl-2.3.xml new file mode 100644 index 000000000..0ad7d641f --- /dev/null +++ b/third_party/nix/doc/manual/release-notes/rl-2.3.xml @@ -0,0 +1,91 @@ +
+ +Release 2.3 (2019-09-04) + +This is primarily a bug fix release. However, it makes some +incompatible changes: + + + + + Nix now uses BSD file locks instead of POSIX file + locks. Because of this, you should not use Nix 2.3 and previous + releases at the same time on a Nix store. + + + + +It also has the following changes: + + + + + builtins.fetchGit's ref + argument now allows specifying an absolute remote ref. + Nix will automatically prefix ref with + refs/heads only if ref doesn't + already begin with refs/. + + + + + The installer now enables sandboxing by default on Linux when the + system has the necessary kernel support. + + + + + The max-jobs setting now defaults to 1. + + + + New builtin functions: + builtins.isPath, + builtins.hashFile. + + + + + The nix command has a new + () flag to + print build log output to stderr, rather than showing the last log + line in the progress bar. To distinguish between concurrent + builds, log lines are prefixed by the name of the package. + + + + + Builds are now executed in a pseudo-terminal, and the + TERM environment variable is set to + xterm-256color. This allows many programs + (e.g. gcc, clang, + cmake) to print colorized log output. + + + + Add convenience flag. This flag + disables substituters; sets the tarball-ttl + setting to infinity (ensuring that any previously downloaded files + are considered current); and disables retrying downloads and sets + the connection timeout to the minimum. This flag is enabled + automatically if there are no configured non-loopback network + interfaces. + + + + Add a post-build-hook setting to run a + program after a build has succeeded. + + + + Add a trace-function-calls setting to log + the duration of Nix function calls to stderr. + + + + +
diff --git a/third_party/nix/doc/manual/schemas.xml b/third_party/nix/doc/manual/schemas.xml new file mode 100644 index 000000000..691a517b9 --- /dev/null +++ b/third_party/nix/doc/manual/schemas.xml @@ -0,0 +1,4 @@ + + + + diff --git a/third_party/nix/local.mk b/third_party/nix/local.mk new file mode 100644 index 000000000..4b380176f --- /dev/null +++ b/third_party/nix/local.mk @@ -0,0 +1,12 @@ +ifeq ($(MAKECMDGOALS), dist) + dist-files += $(shell cat .dist-files) +endif + +dist-files += configure config.h.in nix.spec perl/configure + +clean-files += Makefile.config + +GLOBAL_CXXFLAGS += -I . -I src -I src/libutil -I src/libstore -I src/libmain -I src/libexpr -I src/nix + +$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) diff --git a/third_party/nix/m4/ax_cxx_compile_stdcxx.m4 b/third_party/nix/m4/ax_cxx_compile_stdcxx.m4 new file mode 100644 index 000000000..43087b2e6 --- /dev/null +++ b/third_party/nix/m4/ax_cxx_compile_stdcxx.m4 @@ -0,0 +1,951 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX(VERSION, [ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the specified +# version of the C++ standard. If necessary, add switches to CXX and +# CXXCPP to enable support. VERSION may be '11' (for the C++11 standard) +# or '14' (for the C++14 standard). +# +# The second argument, if specified, indicates whether you insist on an +# extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. +# -std=c++11). If neither is specified, you get whatever works, with +# preference for an extended mode. +# +# The third argument, if specified 'mandatory' or if left unspecified, +# indicates that baseline support for the specified C++ standard is +# required and that the macro should error out if no mode with that +# support is found. If specified 'optional', then configuration proceeds +# regardless, after defining HAVE_CXX${VERSION} if and only if a +# supporting mode is found. +# +# LICENSE +# +# Copyright (c) 2008 Benjamin Kosnik +# Copyright (c) 2012 Zack Weinberg +# Copyright (c) 2013 Roy Stogner +# Copyright (c) 2014, 2015 Google Inc.; contributed by Alexey Sokolov +# Copyright (c) 2015 Paul Norman +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016, 2018 Krzesimir Nowak +# Copyright (c) 2019 Enji Cooper +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 11 + +dnl This macro is based on the code from the AX_CXX_COMPILE_STDCXX_11 macro +dnl (serial version number 13). + +AC_DEFUN([AX_CXX_COMPILE_STDCXX], [dnl + m4_if([$1], [11], [ax_cxx_compile_alternatives="11 0x"], + [$1], [14], [ax_cxx_compile_alternatives="14 1y"], + [$1], [17], [ax_cxx_compile_alternatives="17 1z"], + [m4_fatal([invalid first argument `$1' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$2], [], [], + [$2], [ext], [], + [$2], [noext], [], + [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX])])dnl + m4_if([$3], [], [ax_cxx_compile_cxx$1_required=true], + [$3], [mandatory], [ax_cxx_compile_cxx$1_required=true], + [$3], [optional], [ax_cxx_compile_cxx$1_required=false], + [m4_fatal([invalid third argument `$3' to AX_CXX_COMPILE_STDCXX])]) + AC_LANG_PUSH([C++])dnl + ac_success=no + + m4_if([$2], [noext], [], [dnl + if test x$ac_success = xno; then + for alternative in ${ax_cxx_compile_alternatives}; do + switch="-std=gnu++${alternative}" + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + fi]) + + m4_if([$2], [ext], [], [dnl + if test x$ac_success = xno; then + dnl HP's aCC needs +std=c++11 according to: + dnl http://h21007.www2.hp.com/portal/download/files/unprot/aCxx/PDF_Release_Notes/769149-001.pdf + dnl Cray's crayCC needs "-h std=c++11" + for alternative in ${ax_cxx_compile_alternatives}; do + for switch in -std=c++${alternative} +std=c++${alternative} "-h std=c++${alternative}"; do + cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx$1_$switch]) + AC_CACHE_CHECK(whether $CXX supports C++$1 features with $switch, + $cachevar, + [ac_save_CXX="$CXX" + CXX="$CXX $switch" + AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_testbody_$1])], + [eval $cachevar=yes], + [eval $cachevar=no]) + CXX="$ac_save_CXX"]) + if eval test x\$$cachevar = xyes; then + CXX="$CXX $switch" + if test -n "$CXXCPP" ; then + CXXCPP="$CXXCPP $switch" + fi + ac_success=yes + break + fi + done + if test x$ac_success = xyes; then + break + fi + done + fi]) + AC_LANG_POP([C++]) + if test x$ax_cxx_compile_cxx$1_required = xtrue; then + if test x$ac_success = xno; then + AC_MSG_ERROR([*** A compiler with support for C++$1 language features is required.]) + fi + fi + if test x$ac_success = xno; then + HAVE_CXX$1=0 + AC_MSG_NOTICE([No compiler with C++$1 support was found]) + else + HAVE_CXX$1=1 + AC_DEFINE(HAVE_CXX$1,1, + [define if the compiler supports basic C++$1 syntax]) + fi + AC_SUBST(HAVE_CXX$1) +]) + + +dnl Test body for checking C++11 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_11], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 +) + + +dnl Test body for checking C++14 support + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_14], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 +) + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_17], + _AX_CXX_COMPILE_STDCXX_testbody_new_in_11 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_14 + _AX_CXX_COMPILE_STDCXX_testbody_new_in_17 +) + +dnl Tests for new features in C++11 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_11], [[ + +// If the compiler admits that it is not ready for C++11, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201103L + +#error "This is not a C++11 compiler" + +#else + +namespace cxx11 +{ + + namespace test_static_assert + { + + template + struct check + { + static_assert(sizeof(int) <= sizeof(T), "not big enough"); + }; + + } + + namespace test_final_override + { + + struct Base + { + virtual ~Base() {} + virtual void f() {} + }; + + struct Derived : public Base + { + virtual ~Derived() override {} + virtual void f() override {} + }; + + } + + namespace test_double_right_angle_brackets + { + + template < typename T > + struct check {}; + + typedef check single_type; + typedef check> double_type; + typedef check>> triple_type; + typedef check>>> quadruple_type; + + } + + namespace test_decltype + { + + int + f() + { + int a = 1; + decltype(a) b = 2; + return a + b; + } + + } + + namespace test_type_deduction + { + + template < typename T1, typename T2 > + struct is_same + { + static const bool value = false; + }; + + template < typename T > + struct is_same + { + static const bool value = true; + }; + + template < typename T1, typename T2 > + auto + add(T1 a1, T2 a2) -> decltype(a1 + a2) + { + return a1 + a2; + } + + int + test(const int c, volatile int v) + { + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == false, ""); + auto ac = c; + auto av = v; + auto sumi = ac + av + 'x'; + auto sumf = ac + av + 1.0; + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == true, ""); + static_assert(is_same::value == false, ""); + static_assert(is_same::value == true, ""); + return (sumf > 0.0) ? sumi : add(c, v); + } + + } + + namespace test_noexcept + { + + int f() { return 0; } + int g() noexcept { return 0; } + + static_assert(noexcept(f()) == false, ""); + static_assert(noexcept(g()) == true, ""); + + } + + namespace test_constexpr + { + + template < typename CharT > + unsigned long constexpr + strlen_c_r(const CharT *const s, const unsigned long acc) noexcept + { + return *s ? strlen_c_r(s + 1, acc + 1) : acc; + } + + template < typename CharT > + unsigned long constexpr + strlen_c(const CharT *const s) noexcept + { + return strlen_c_r(s, 0UL); + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("1") == 1UL, ""); + static_assert(strlen_c("example") == 7UL, ""); + static_assert(strlen_c("another\0example") == 7UL, ""); + + } + + namespace test_rvalue_references + { + + template < int N > + struct answer + { + static constexpr int value = N; + }; + + answer<1> f(int&) { return answer<1>(); } + answer<2> f(const int&) { return answer<2>(); } + answer<3> f(int&&) { return answer<3>(); } + + void + test() + { + int i = 0; + const int c = 0; + static_assert(decltype(f(i))::value == 1, ""); + static_assert(decltype(f(c))::value == 2, ""); + static_assert(decltype(f(0))::value == 3, ""); + } + + } + + namespace test_uniform_initialization + { + + struct test + { + static const int zero {}; + static const int one {1}; + }; + + static_assert(test::zero == 0, ""); + static_assert(test::one == 1, ""); + + } + + namespace test_lambdas + { + + void + test1() + { + auto lambda1 = [](){}; + auto lambda2 = lambda1; + lambda1(); + lambda2(); + } + + int + test2() + { + auto a = [](int i, int j){ return i + j; }(1, 2); + auto b = []() -> int { return '0'; }(); + auto c = [=](){ return a + b; }(); + auto d = [&](){ return c; }(); + auto e = [a, &b](int x) mutable { + const auto identity = [](int y){ return y; }; + for (auto i = 0; i < a; ++i) + a += b--; + return x + identity(a + b); + }(0); + return a + b + c + d + e; + } + + int + test3() + { + const auto nullary = [](){ return 0; }; + const auto unary = [](int x){ return x; }; + using nullary_t = decltype(nullary); + using unary_t = decltype(unary); + const auto higher1st = [](nullary_t f){ return f(); }; + const auto higher2nd = [unary](nullary_t f1){ + return [unary, f1](unary_t f2){ return f2(unary(f1())); }; + }; + return higher1st(nullary) + higher2nd(nullary)(unary); + } + + } + + namespace test_variadic_templates + { + + template + struct sum; + + template + struct sum + { + static constexpr auto value = N0 + sum::value; + }; + + template <> + struct sum<> + { + static constexpr auto value = 0; + }; + + static_assert(sum<>::value == 0, ""); + static_assert(sum<1>::value == 1, ""); + static_assert(sum<23>::value == 23, ""); + static_assert(sum<1, 2>::value == 3, ""); + static_assert(sum<5, 5, 11>::value == 21, ""); + static_assert(sum<2, 3, 5, 7, 11, 13>::value == 41, ""); + + } + + // http://stackoverflow.com/questions/13728184/template-aliases-and-sfinae + // Clang 3.1 fails with headers of libstd++ 4.8.3 when using std::function + // because of this. + namespace test_template_alias_sfinae + { + + struct foo {}; + + template + using member = typename T::member_type; + + template + void func(...) {} + + template + void func(member*) {} + + void test(); + + void test() { func(0); } + + } + +} // namespace cxx11 + +#endif // __cplusplus >= 201103L + +]]) + + +dnl Tests for new features in C++14 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_14], [[ + +// If the compiler admits that it is not ready for C++14, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201402L + +#error "This is not a C++14 compiler" + +#else + +namespace cxx14 +{ + + namespace test_polymorphic_lambdas + { + + int + test() + { + const auto lambda = [](auto&&... args){ + const auto istiny = [](auto x){ + return (sizeof(x) == 1UL) ? 1 : 0; + }; + const int aretiny[] = { istiny(args)... }; + return aretiny[0]; + }; + return lambda(1, 1L, 1.0f, '1'); + } + + } + + namespace test_binary_literals + { + + constexpr auto ivii = 0b0000000000101010; + static_assert(ivii == 42, "wrong value"); + + } + + namespace test_generalized_constexpr + { + + template < typename CharT > + constexpr unsigned long + strlen_c(const CharT *const s) noexcept + { + auto length = 0UL; + for (auto p = s; *p; ++p) + ++length; + return length; + } + + static_assert(strlen_c("") == 0UL, ""); + static_assert(strlen_c("x") == 1UL, ""); + static_assert(strlen_c("test") == 4UL, ""); + static_assert(strlen_c("another\0test") == 7UL, ""); + + } + + namespace test_lambda_init_capture + { + + int + test() + { + auto x = 0; + const auto lambda1 = [a = x](int b){ return a + b; }; + const auto lambda2 = [a = lambda1(x)](){ return a; }; + return lambda2(); + } + + } + + namespace test_digit_separators + { + + constexpr auto ten_million = 100'000'000; + static_assert(ten_million == 100000000, ""); + + } + + namespace test_return_type_deduction + { + + auto f(int& x) { return x; } + decltype(auto) g(int& x) { return x; } + + template < typename T1, typename T2 > + struct is_same + { + static constexpr auto value = false; + }; + + template < typename T > + struct is_same + { + static constexpr auto value = true; + }; + + int + test() + { + auto x = 0; + static_assert(is_same::value, ""); + static_assert(is_same::value, ""); + return x; + } + + } + +} // namespace cxx14 + +#endif // __cplusplus >= 201402L + +]]) + + +dnl Tests for new features in C++17 + +m4_define([_AX_CXX_COMPILE_STDCXX_testbody_new_in_17], [[ + +// If the compiler admits that it is not ready for C++17, why torture it? +// Hopefully, this will speed up the test. + +#ifndef __cplusplus + +#error "This is not a C++ compiler" + +#elif __cplusplus < 201703L + +#error "This is not a C++17 compiler" + +#else + +#include +#include +#include + +namespace cxx17 +{ + + namespace test_constexpr_lambdas + { + + constexpr int foo = [](){return 42;}(); + + } + + namespace test::nested_namespace::definitions + { + + } + + namespace test_fold_expression + { + + template + int multiply(Args... args) + { + return (args * ... * 1); + } + + template + bool all(Args... args) + { + return (args && ...); + } + + } + + namespace test_extended_static_assert + { + + static_assert (true); + + } + + namespace test_auto_brace_init_list + { + + auto foo = {5}; + auto bar {5}; + + static_assert(std::is_same, decltype(foo)>::value); + static_assert(std::is_same::value); + } + + namespace test_typename_in_template_template_parameter + { + + template typename X> struct D; + + } + + namespace test_fallthrough_nodiscard_maybe_unused_attributes + { + + int f1() + { + return 42; + } + + [[nodiscard]] int f2() + { + [[maybe_unused]] auto unused = f1(); + + switch (f1()) + { + case 17: + f1(); + [[fallthrough]]; + case 42: + f1(); + } + return f1(); + } + + } + + namespace test_extended_aggregate_initialization + { + + struct base1 + { + int b1, b2 = 42; + }; + + struct base2 + { + base2() { + b3 = 42; + } + int b3; + }; + + struct derived : base1, base2 + { + int d; + }; + + derived d1 {{1, 2}, {}, 4}; // full initialization + derived d2 {{}, {}, 4}; // value-initialized bases + + } + + namespace test_general_range_based_for_loop + { + + struct iter + { + int i; + + int& operator* () + { + return i; + } + + const int& operator* () const + { + return i; + } + + iter& operator++() + { + ++i; + return *this; + } + }; + + struct sentinel + { + int i; + }; + + bool operator== (const iter& i, const sentinel& s) + { + return i.i == s.i; + } + + bool operator!= (const iter& i, const sentinel& s) + { + return !(i == s); + } + + struct range + { + iter begin() const + { + return {0}; + } + + sentinel end() const + { + return {5}; + } + }; + + void f() + { + range r {}; + + for (auto i : r) + { + [[maybe_unused]] auto v = i; + } + } + + } + + namespace test_lambda_capture_asterisk_this_by_value + { + + struct t + { + int i; + int foo() + { + return [*this]() + { + return i; + }(); + } + }; + + } + + namespace test_enum_class_construction + { + + enum class byte : unsigned char + {}; + + byte foo {42}; + + } + + namespace test_constexpr_if + { + + template + int f () + { + if constexpr(cond) + { + return 13; + } + else + { + return 42; + } + } + + } + + namespace test_selection_statement_with_initializer + { + + int f() + { + return 13; + } + + int f2() + { + if (auto i = f(); i > 0) + { + return 3; + } + + switch (auto i = f(); i + 4) + { + case 17: + return 2; + + default: + return 1; + } + } + + } + + namespace test_template_argument_deduction_for_class_templates + { + + template + struct pair + { + pair (T1 p1, T2 p2) + : m1 {p1}, + m2 {p2} + {} + + T1 m1; + T2 m2; + }; + + void f() + { + [[maybe_unused]] auto p = pair{13, 42u}; + } + + } + + namespace test_non_type_auto_template_parameters + { + + template + struct B + {}; + + B<5> b1; + B<'a'> b2; + + } + + namespace test_structured_bindings + { + + int arr[2] = { 1, 2 }; + std::pair pr = { 1, 2 }; + + auto f1() -> int(&)[2] + { + return arr; + } + + auto f2() -> std::pair& + { + return pr; + } + + struct S + { + int x1 : 2; + volatile double y1; + }; + + S f3() + { + return {}; + } + + auto [ x1, y1 ] = f1(); + auto& [ xr1, yr1 ] = f1(); + auto [ x2, y2 ] = f2(); + auto& [ xr2, yr2 ] = f2(); + const auto [ x3, y3 ] = f3(); + + } + + namespace test_exception_spec_type_system + { + + struct Good {}; + struct Bad {}; + + void g1() noexcept; + void g2(); + + template + Bad + f(T*, T*); + + template + Good + f(T1*, T2*); + + static_assert (std::is_same_v); + + } + + namespace test_inline_variables + { + + template void f(T) + {} + + template inline T g(T) + { + return T{}; + } + + template<> inline void f<>(int) + {} + + template<> int g<>(int) + { + return 5; + } + + } + +} // namespace cxx17 + +#endif // __cplusplus < 201703L + +]]) diff --git a/third_party/nix/m4/ax_cxx_compile_stdcxx_17.m4 b/third_party/nix/m4/ax_cxx_compile_stdcxx_17.m4 new file mode 100644 index 000000000..a68341717 --- /dev/null +++ b/third_party/nix/m4/ax_cxx_compile_stdcxx_17.m4 @@ -0,0 +1,35 @@ +# ============================================================================= +# https://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_17.html +# ============================================================================= +# +# SYNOPSIS +# +# AX_CXX_COMPILE_STDCXX_17([ext|noext], [mandatory|optional]) +# +# DESCRIPTION +# +# Check for baseline language coverage in the compiler for the C++17 +# standard; if necessary, add switches to CXX and CXXCPP to enable +# support. +# +# This macro is a convenience alias for calling the AX_CXX_COMPILE_STDCXX +# macro with the version set to C++17. The two optional arguments are +# forwarded literally as the second and third argument respectively. +# Please see the documentation for the AX_CXX_COMPILE_STDCXX macro for +# more information. If you want to use this macro, you also need to +# download the ax_cxx_compile_stdcxx.m4 file. +# +# LICENSE +# +# Copyright (c) 2015 Moritz Klammler +# Copyright (c) 2016 Krzesimir Nowak +# +# Copying and distribution of this file, with or without modification, are +# permitted in any medium without royalty provided the copyright notice +# and this notice are preserved. This file is offered as-is, without any +# warranty. + +#serial 2 + +AX_REQUIRE_DEFINED([AX_CXX_COMPILE_STDCXX]) +AC_DEFUN([AX_CXX_COMPILE_STDCXX_17], [AX_CXX_COMPILE_STDCXX([17], [$1], [$2])]) diff --git a/third_party/nix/maintainers/upload-release.pl b/third_party/nix/maintainers/upload-release.pl new file mode 100755 index 000000000..1cdf5ed16 --- /dev/null +++ b/third_party/nix/maintainers/upload-release.pl @@ -0,0 +1,156 @@ +#! /usr/bin/env nix-shell +#! nix-shell -i perl -p perl perlPackages.LWPUserAgent perlPackages.LWPProtocolHttps perlPackages.FileSlurp gnupg1 + +use strict; +use Data::Dumper; +use File::Basename; +use File::Path; +use File::Slurp; +use File::Copy; +use JSON::PP; +use LWP::UserAgent; + +my $evalId = $ARGV[0] or die "Usage: $0 EVAL-ID\n"; + +my $releasesDir = "/home/eelco/mnt/releases"; +my $nixpkgsDir = "/home/eelco/Dev/nixpkgs-pristine"; + +# FIXME: cut&paste from nixos-channel-scripts. +sub fetch { + my ($url, $type) = @_; + + my $ua = LWP::UserAgent->new; + $ua->default_header('Accept', $type) if defined $type; + + my $response = $ua->get($url); + die "could not download $url: ", $response->status_line, "\n" unless $response->is_success; + + return $response->decoded_content; +} + +my $evalUrl = "https://hydra.nixos.org/eval/$evalId"; +my $evalInfo = decode_json(fetch($evalUrl, 'application/json')); +#print Dumper($evalInfo); + +my $nixRev = $evalInfo->{jobsetevalinputs}->{nix}->{revision} or die; + +my $tarballInfo = decode_json(fetch("$evalUrl/job/tarball", 'application/json')); + +my $releaseName = $tarballInfo->{releasename}; +$releaseName =~ /nix-(.*)$/ or die; +my $version = $1; + +print STDERR "Nix revision is $nixRev, version is $version\n"; + +File::Path::make_path($releasesDir); +if (system("mountpoint -q $releasesDir") != 0) { + system("sshfs hydra-mirror:/releases $releasesDir") == 0 or die; +} + +my $releaseDir = "$releasesDir/nix/$releaseName"; +File::Path::make_path($releaseDir); + +sub downloadFile { + my ($jobName, $productNr, $dstName) = @_; + + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + + my $srcFile = $buildInfo->{buildproducts}->{$productNr}->{path} or die "job '$jobName' lacks product $productNr\n"; + $dstName //= basename($srcFile); + my $dstFile = "$releaseDir/" . $dstName; + + if (! -e $dstFile) { + print STDERR "downloading $srcFile to $dstFile...\n"; + system("NIX_REMOTE=https://cache.nixos.org/ nix cat-store '$srcFile' > '$dstFile.tmp'") == 0 + or die "unable to fetch $srcFile\n"; + rename("$dstFile.tmp", $dstFile) or die; + } + + my $sha256_expected = $buildInfo->{buildproducts}->{$productNr}->{sha256hash} or die; + my $sha256_actual = `nix hash-file --base16 --type sha256 '$dstFile'`; + chomp $sha256_actual; + if ($sha256_expected ne $sha256_actual) { + print STDERR "file $dstFile is corrupt, got $sha256_actual, expected $sha256_expected\n"; + exit 1; + } + + write_file("$dstFile.sha256", $sha256_expected); + + if (! -e "$dstFile.asc") { + system("gpg2 --detach-sign --armor $dstFile") == 0 or die "unable to sign $dstFile\n"; + } + + return ($dstFile, $sha256_expected); +} + +downloadFile("tarball", "2"); # .tar.bz2 +my ($tarball, $tarballHash) = downloadFile("tarball", "3"); # .tar.xz +downloadFile("binaryTarball.i686-linux", "1"); +downloadFile("binaryTarball.x86_64-linux", "1"); +downloadFile("binaryTarball.aarch64-linux", "1"); +downloadFile("binaryTarball.x86_64-darwin", "1"); +downloadFile("installerScript", "1"); + +exit if $version =~ /pre/; + +# Update Nixpkgs in a very hacky way. +system("cd $nixpkgsDir && git pull") == 0 or die; +my $oldName = `nix-instantiate --eval $nixpkgsDir -A nix.name`; chomp $oldName; +my $oldHash = `nix-instantiate --eval $nixpkgsDir -A nix.src.outputHash`; chomp $oldHash; +print STDERR "old stable version in Nixpkgs = $oldName / $oldHash\n"; + +my $fn = "$nixpkgsDir/pkgs/tools/package-management/nix/default.nix"; +my $oldFile = read_file($fn); +$oldFile =~ s/$oldName/"$releaseName"/g; +$oldFile =~ s/$oldHash/"$tarballHash"/g; +write_file($fn, $oldFile); + +$oldName =~ s/nix-//g; +$oldName =~ s/"//g; + +sub getStorePath { + my ($jobName) = @_; + my $buildInfo = decode_json(fetch("$evalUrl/job/$jobName", 'application/json')); + die unless $buildInfo->{buildproducts}->{1}->{type} eq "nix-build"; + return $buildInfo->{buildproducts}->{1}->{path}; +} + +write_file("$nixpkgsDir/nixos/modules/installer/tools/nix-fallback-paths.nix", + "{\n" . + " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . + "}\n"); + +system("cd $nixpkgsDir && git commit -a -m 'nix: $oldName -> $version'") == 0 or die; + +# Extract the HTML manual. +File::Path::make_path("$releaseDir/manual"); + +system("tar xvf $tarball --strip-components=3 -C $releaseDir/manual --wildcards '*/doc/manual/*.html' '*/doc/manual/*.css' '*/doc/manual/*.gif' '*/doc/manual/*.png'") == 0 or die; + +if (! -e "$releaseDir/manual/index.html") { + symlink("manual.html", "$releaseDir/manual/index.html") or die; +} + +# Update the "latest" symlink. +symlink("$releaseName", "$releasesDir/nix/latest-tmp") or die; +rename("$releasesDir/nix/latest-tmp", "$releasesDir/nix/latest") or die; + +# Tag the release in Git. +chdir("/home/eelco/Dev/nix-pristine") or die; +system("git remote update origin") == 0 or die; +system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die; + +# Update the website. +my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine"; + +system("cd $siteDir && git pull") == 0 or die; + +write_file("$siteDir/nix-release.tt", + "[%-\n" . + "latestNixVersion = \"$version\"\n" . + "-%]\n"); + +system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die; diff --git a/third_party/nix/misc/launchd/local.mk b/third_party/nix/misc/launchd/local.mk new file mode 100644 index 000000000..0ba722efb --- /dev/null +++ b/third_party/nix/misc/launchd/local.mk @@ -0,0 +1,5 @@ +ifeq ($(OS), Darwin) + + $(eval $(call install-data-in, $(d)/org.nixos.nix-daemon.plist, $(prefix)/Library/LaunchDaemons)) + +endif diff --git a/third_party/nix/misc/launchd/org.nixos.nix-daemon.plist.in b/third_party/nix/misc/launchd/org.nixos.nix-daemon.plist.in new file mode 100644 index 000000000..9f26296a9 --- /dev/null +++ b/third_party/nix/misc/launchd/org.nixos.nix-daemon.plist.in @@ -0,0 +1,27 @@ + + + + + EnvironmentVariables + + OBJC_DISABLE_INITIALIZE_FORK_SAFETY + YES + + Label + org.nixos.nix-daemon + KeepAlive + + RunAtLoad + + ProgramArguments + + /bin/sh + -c + /bin/wait4path /nix/var/nix/profiles/default/bin/nix-daemon && /nix/var/nix/profiles/default/bin/nix-daemon + + StandardErrorPath + /var/log/nix-daemon.log + StandardOutPath + /dev/null + + diff --git a/third_party/nix/misc/systemd/local.mk b/third_party/nix/misc/systemd/local.mk new file mode 100644 index 000000000..004549fd2 --- /dev/null +++ b/third_party/nix/misc/systemd/local.mk @@ -0,0 +1,5 @@ +ifeq ($(OS), Linux) + + $(foreach n, nix-daemon.socket nix-daemon.service, $(eval $(call install-file-in, $(d)/$(n), $(prefix)/lib/systemd/system, 0644))) + +endif diff --git a/third_party/nix/misc/systemd/nix-daemon.service.in b/third_party/nix/misc/systemd/nix-daemon.service.in new file mode 100644 index 000000000..25655204d --- /dev/null +++ b/third_party/nix/misc/systemd/nix-daemon.service.in @@ -0,0 +1,12 @@ +[Unit] +Description=Nix Daemon +RequiresMountsFor=@storedir@ +RequiresMountsFor=@localstatedir@ +ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket + +[Service] +ExecStart=@@bindir@/nix-daemon nix-daemon --daemon +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/third_party/nix/misc/systemd/nix-daemon.socket.in b/third_party/nix/misc/systemd/nix-daemon.socket.in new file mode 100644 index 000000000..9ed39ffe6 --- /dev/null +++ b/third_party/nix/misc/systemd/nix-daemon.socket.in @@ -0,0 +1,11 @@ +[Unit] +Description=Nix Daemon Socket +Before=multi-user.target +RequiresMountsFor=@storedir@ +ConditionPathIsReadWrite=@localstatedir@/nix/daemon-socket + +[Socket] +ListenStream=@localstatedir@/nix/daemon-socket/socket + +[Install] +WantedBy=sockets.target diff --git a/third_party/nix/misc/systemv/nix-daemon b/third_party/nix/misc/systemv/nix-daemon new file mode 100755 index 000000000..fea537167 --- /dev/null +++ b/third_party/nix/misc/systemv/nix-daemon @@ -0,0 +1,113 @@ +#!/bin/sh +# +# nix-daemon: Starts the nix package manager daemon +# +# chkconfig: 345 24 02 +# description: This is a daemon which enable the multi-user mode +# of the nix package manager. +# processname: nix-daemon +# pidfile: /var/run/nix/nix-daemon.pid + +### BEGIN INIT INFO +# Required-Start: +# Required-Stop: +# Should-Start: +# Should-Stop: +# Default-Start: 3 4 5 +# Default-Stop: 0 1 2 6 +# Short-Description: Starts the nix daemon +# Description: This is a daemon which enable the multi-user mode +# of the nix package manager. +### END INIT INFO + +NIX_DAEMON_BIN=/usr/bin/nix-daemon +#NIX_DAEMON_USER="root" +NIX_DAEMON_USER="nix-daemon" +NIX_DAEMON_OPTS="--daemon" + +umask 0022 + +if [ "$1" = 'status' ]; then + test -x $NIX_DAEMON_BIN || exit 4 +else + test -x $NIX_DAEMON_BIN || exit 5 +fi + +# Source function library. +. /etc/init.d/functions + +LOCKFILE=/var/lock/subsys/nix-daemon +RUNDIR=/var/run/nix +PIDFILE=${RUNDIR}/nix-daemon.pid +RETVAL=0 + +base=${0##*/} + +start() { + + mkdir -p ${RUNDIR} + chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} + + echo -n $"Starting nix daemon... " + + daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS + RETVAL=$? + echo + [ $RETVAL -eq 0 ] && touch ${LOCKFILE} + return $RETVAL +} + +stop() { + echo -n $"Shutting down nix daemon: " + killproc -p ${PIDFILE} $NIX_DAEMON_BIN + RETVAL=$? + [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} + echo + return $RETVAL +} + +reload() { + echo -n $"Reloading nix daemon... " + killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP + RETVAL=$? + echo + return $RETVAL +} + +restart() { + stop + start +} + +RETVAL=0 + +# caller switch +case "$1" in + start) + start + ;; + stop) + stop + ;; + status) + status -p ${PIDFILE} $NIX_DAEMON_BIN + RETVAL=$? + ;; + restart) + restart + ;; + reload) + reload + ;; + condrestart) + if [ -f $LOCKFILE ]; then + restart + fi + ;; + *) + echo $"Usage: $0 {start|stop|status|restart|condrestart}" + exit 2 + ;; +esac + +exit $RETVAL diff --git a/third_party/nix/misc/upstart/local.mk b/third_party/nix/misc/upstart/local.mk new file mode 100644 index 000000000..a73dc061e --- /dev/null +++ b/third_party/nix/misc/upstart/local.mk @@ -0,0 +1,5 @@ +ifeq ($(OS), Linux) + + $(foreach n, nix-daemon.conf, $(eval $(call install-file-in, $(d)/$(n), $(sysconfdir)/init, 0644))) + +endif diff --git a/third_party/nix/misc/upstart/nix-daemon.conf.in b/third_party/nix/misc/upstart/nix-daemon.conf.in new file mode 100644 index 000000000..0e806edbd --- /dev/null +++ b/third_party/nix/misc/upstart/nix-daemon.conf.in @@ -0,0 +1,5 @@ +description "Nix Daemon" +start on filesystem +stop on shutdown +respawn +exec @bindir@/nix-daemon --daemon diff --git a/third_party/nix/mk/clean.mk b/third_party/nix/mk/clean.mk new file mode 100644 index 000000000..ce9afb3b0 --- /dev/null +++ b/third_party/nix/mk/clean.mk @@ -0,0 +1,11 @@ +clean-files := + +clean: + $(suppress) rm -fv -- $(clean-files) + +dryclean: + @for i in $(clean-files); do if [ -e $$i ]; then echo $$i; fi; done | sort + +print-top-help += \ + echo " clean: Delete generated files"; \ + echo " dryclean: Show what files would be deleted by 'make clean'"; diff --git a/third_party/nix/mk/dist.mk b/third_party/nix/mk/dist.mk new file mode 100644 index 000000000..794b27771 --- /dev/null +++ b/third_party/nix/mk/dist.mk @@ -0,0 +1,17 @@ +ifdef PACKAGE_NAME + +dist-name = $(PACKAGE_NAME)-$(PACKAGE_VERSION) + +dist: $(dist-name).tar.bz2 $(dist-name).tar.xz + +$(dist-name).tar.bz2: $(dist-files) + $(trace-gen) tar cfj $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,' + +$(dist-name).tar.xz: $(dist-files) + $(trace-gen) tar cfJ $@ $(sort $(dist-files)) --transform 's,^,$(dist-name)/,' + +clean-files += $(dist-name).tar.bz2 $(dist-name).tar.xz + +print-top-help += echo " dist: Generate a source distribution"; + +endif diff --git a/third_party/nix/mk/functions.mk b/third_party/nix/mk/functions.mk new file mode 100644 index 000000000..c48775db8 --- /dev/null +++ b/third_party/nix/mk/functions.mk @@ -0,0 +1,14 @@ +# Utility function for recursively finding files, e.g. +# ‘$(call rwildcard, path/to/dir, *.c *.h)’. +rwildcard=$(foreach d,$(wildcard $1*),$(call rwildcard,$d/,$2) $(filter $(subst *,%,$2),$d)) + +# Given a file name, produce the corresponding dependency file +# (e.g. ‘foo/bar.o’ becomes ‘foo/.bar.o.dep’). +filename-to-dep = $(dir $1).$(notdir $1).dep + +# Return the full path to a program by looking it up in $PATH, or the +# empty string if not found. +find-program = $(shell for i in $$(IFS=: ; echo $$PATH); do p=$$i/$(strip $1); if [ -e $$p ]; then echo $$p; break; fi; done) + +# Ensure that the given string ends in a single slash. +add-trailing-slash = $(patsubst %/,%,$(1))/ diff --git a/third_party/nix/mk/install.mk b/third_party/nix/mk/install.mk new file mode 100644 index 000000000..dad0fd853 --- /dev/null +++ b/third_party/nix/mk/install.mk @@ -0,0 +1,62 @@ +# Add a rule for creating $(1) as a directory. This template may be +# called multiple times for the same directory. +define create-dir + _i := $$(call add-trailing-slash, $(DESTDIR)$$(strip $(1))) + ifndef $$(_i)_SEEN + $$(_i)_SEEN = 1 + $$(_i): + $$(trace-mkdir) install -d "$$@" + endif +endef + + +# Add a rule for installing file $(1) as file $(2) with mode $(3). +# The directory containing $(2) will be created automatically. +define install-file-as + + _i := $(DESTDIR)$$(strip $(2)) + + install: $$(_i) + + $$(_i): $(1) | $$(dir $$(_i)) + $$(trace-install) install -m $(3) $(1) "$$@" + + $$(eval $$(call create-dir, $$(dir $(2)))) + +endef + + +# Add a rule for installing file $(1) in directory $(2) with mode +# $(3). The directory will be created automatically. +define install-file-in + $$(eval $$(call install-file-as,$(1),$(2)/$$(notdir $(1)),$(3))) +endef + + +define install-program-in + $$(eval $$(call install-file-in,$(1),$(2),0755)) +endef + + +define install-data-in + $$(eval $$(call install-file-in,$(1),$(2),0644)) +endef + + +# Install a symlink from $(2) to $(1). Note that $(1) need not exist. +define install-symlink + + _i := $(DESTDIR)$$(strip $(2)) + + install: $$(_i) + + $$(_i): | $$(dir $$(_i)) + $$(trace-install) ln -sfn $(1) "$$@" + + $$(eval $$(call create-dir, $$(dir $(2)))) + +endef + + +print-top-help += \ + echo " install: Install into \$$(prefix) (currently set to '$(prefix)')"; diff --git a/third_party/nix/mk/jars.mk b/third_party/nix/mk/jars.mk new file mode 100644 index 000000000..c8513e664 --- /dev/null +++ b/third_party/nix/mk/jars.mk @@ -0,0 +1,36 @@ +define build-jar + + $(1)_NAME ?= $(1) + + _d := $$(strip $$($(1)_DIR)) + + $(1)_PATH := $$(_d)/$$($(1)_NAME).jar + + $(1)_TMPDIR := $$(_d)/.$$($(1)_NAME).jar.tmp + + _jars := $$(foreach jar, $$($(1)_JARS), $$($$(jar)_PATH)) + + $$($(1)_PATH): $$($(1)_SOURCES) $$(_jars) $$($(1)_EXTRA_DEPS)| $$($(1)_ORDER_AFTER) + @rm -rf $$($(1)_TMPDIR) + @mkdir -p $$($(1)_TMPDIR) + $$(trace-javac) javac $(GLOBAL_JAVACFLAGS) $$($(1)_JAVACFLAGS) -d $$($(1)_TMPDIR) \ + $$(foreach fn, $$($(1)_SOURCES), '$$(fn)') \ + -cp "$$(subst $$(space),,$$(foreach jar,$$($(1)_JARS),$$($$(jar)_PATH):))$$$$CLASSPATH" + @echo -e '$$(subst $$(newline),\n,$$($(1)_MANIFEST))' > $$($(1)_PATH).manifest + $$(trace-jar) jar cfm $$($(1)_PATH) $$($(1)_PATH).manifest -C $$($(1)_TMPDIR) . + @rm $$($(1)_PATH).manifest + @rm -rf $$($(1)_TMPDIR) + + $(1)_INSTALL_DIR ?= $$(jardir) + + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$$($(1)_NAME).jar + + $$(eval $$(call install-file-as, $$($(1)_PATH), $$($(1)_INSTALL_PATH), 0644)) + + install: $$($(1)_INSTALL_PATH) + + jars-list += $$($(1)_PATH) + + clean-files += $$($(1)_PATH) + +endef diff --git a/third_party/nix/mk/lib.mk b/third_party/nix/mk/lib.mk new file mode 100644 index 000000000..1da51d879 --- /dev/null +++ b/third_party/nix/mk/lib.mk @@ -0,0 +1,160 @@ +default: all + + +# Get rid of default suffixes. FIXME: is this a good idea? +.SUFFIXES: + + +# Initialise some variables. +bin-scripts := +noinst-scripts := +man-pages := +install-tests := +dist-files := +OS = $(shell uname -s) + + +# Hack to define a literal space. +space := +space += + + +# Hack to define a literal newline. +define newline + + +endef + + +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +jardir ?= $(datadir)/java +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + + +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ +else + buildprefix = +endif + + +# Pass -fPIC if we're building dynamic libraries. +BUILD_SHARED_LIBS ?= 1 + +ifeq ($(BUILD_SHARED_LIBS), 1) + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + GLOBAL_CFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE + GLOBAL_CXXFLAGS += -U__STRICT_ANSI__ -D_GNU_SOURCE + else + GLOBAL_CFLAGS += -fPIC + GLOBAL_CXXFLAGS += -fPIC + endif + ifneq ($(OS), Darwin) + ifneq ($(OS), SunOS) + ifneq ($(OS), FreeBSD) + GLOBAL_LDFLAGS += -Wl,--no-copy-dt-needed-entries + endif + endif + endif + SET_RPATH_TO_LIBS ?= 1 +endif + +# Pass -g if we want debug info. +BUILD_DEBUG ?= 1 + +ifeq ($(BUILD_DEBUG), 1) + GLOBAL_CFLAGS += -g + GLOBAL_CXXFLAGS += -g + GLOBAL_JAVACFLAGS += -g +endif + + +include mk/functions.mk +include mk/tracing.mk +include mk/clean.mk +include mk/install.mk +include mk/libraries.mk +include mk/programs.mk +include mk/jars.mk +include mk/patterns.mk +include mk/templates.mk +include mk/tests.mk + + +# Include all sub-Makefiles. +define include-sub-makefile + d := $$(patsubst %/,%,$$(dir $(1))) + include $(1) +endef + +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) + + +# Instantiate stuff. +$(foreach lib, $(libraries), $(eval $(call build-library,$(lib)))) +$(foreach prog, $(programs), $(eval $(call build-program,$(prog)))) +$(foreach jar, $(jars), $(eval $(call build-jar,$(jar)))) +$(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(bindir)))) +$(foreach script, $(bin-scripts), $(eval programs-list += $(script))) +$(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) +$(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) +$(foreach test, $(install-tests), $(eval $(call run-install-test,$(test)))) +$(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) + + +include mk/dist.mk + + +.PHONY: default all man help + +all: $(programs-list) $(libs-list) $(jars-list) $(man-pages) + +man: $(man-pages) + + +help: + @echo "The following targets are available:" + @echo "" + @echo " default: Build default targets" +ifdef man-pages + @echo " man: Generate manual pages" +endif + @$(print-top-help) +ifdef programs-list + @echo "" + @echo "The following programs can be built:" + @echo "" + @for i in $(programs-list); do echo " $$i"; done +endif +ifdef libs-list + @echo "" + @echo "The following libraries can be built:" + @echo "" + @for i in $(libs-list); do echo " $$i"; done +endif +ifdef jars-list + @echo "" + @echo "The following JARs can be built:" + @echo "" + @for i in $(jars-list); do echo " $$i"; done +endif + @echo "" + @echo "The following variables control the build:" + @echo "" + @echo " BUILD_SHARED_LIBS ($(BUILD_SHARED_LIBS)): Whether to build shared libraries" + @echo " BUILD_DEBUG ($(BUILD_DEBUG)): Whether to include debug symbols" + @echo " CC ($(CC)): C compiler to be used" + @echo " CFLAGS: Flags for the C compiler" + @echo " CXX ($(CXX)): C++ compiler to be used" + @echo " CXXFLAGS: Flags for the C++ compiler" + @$(print-var-help) diff --git a/third_party/nix/mk/libraries.mk b/third_party/nix/mk/libraries.mk new file mode 100644 index 000000000..307e29b9d --- /dev/null +++ b/third_party/nix/mk/libraries.mk @@ -0,0 +1,162 @@ +libs-list := + +ifeq ($(OS), Darwin) + SO_EXT = dylib +else + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + SO_EXT = dll + else + SO_EXT = so + endif +endif + +# Build a library with symbolic name $(1). The library is defined by +# various variables prefixed by ‘$(1)_’: +# +# - $(1)_NAME: the name of the library (e.g. ‘libfoo’); defaults to +# $(1). +# +# - $(1)_DIR: the directory where the (non-installed) library will be +# placed. +# +# - $(1)_SOURCES: the source files of the library. +# +# - $(1)_CFLAGS: additional C compiler flags. +# +# - $(1)_CXXFLAGS: additional C++ compiler flags. +# +# - $(1)_ORDER_AFTER: a set of targets on which the object files of +# this libraries will have an order-only dependency. +# +# - $(1)_LIBS: the symbolic names of other libraries on which this +# library depends. +# +# - $(1)_ALLOW_UNDEFINED: if set, the library is allowed to have +# undefined symbols. Has no effect for static libraries. +# +# - $(1)_LDFLAGS: additional linker flags. +# +# - $(1)_LDFLAGS_PROPAGATED: additional linker flags, also propagated +# to the linking of programs/libraries that use this library. +# +# - $(1)_FORCE_INSTALL: if defined, the library will be installed even +# if it's not needed (i.e. dynamically linked) by a program. +# +# - $(1)_INSTALL_DIR: the directory where the library will be +# installed. Defaults to $(libdir). +# +# - $(1)_EXCLUDE_FROM_LIBRARY_LIST: if defined, the library will not +# be automatically marked as a dependency of the top-level all +# target andwill not be listed in the make help output. This is +# useful for libraries built solely for testing, for example. +# +# - BUILD_SHARED_LIBS: if equal to ‘1’, a dynamic library will be +# built, otherwise a static library. +define build-library + $(1)_NAME ?= $(1) + _d := $(buildprefix)$$(strip $$($(1)_DIR)) + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) + _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) + + ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + $(1)_INSTALL_DIR ?= $$(bindir) + else + $(1)_INSTALL_DIR ?= $$(libdir) + endif + + $(1)_LDFLAGS_USE := + $(1)_LDFLAGS_USE_INSTALLED := + + $$(eval $$(call create-dir, $$(_d))) + + ifeq ($(BUILD_SHARED_LIBS), 1) + + ifdef $(1)_ALLOW_UNDEFINED + ifeq ($(OS), Darwin) + $(1)_LDFLAGS += -undefined suppress -flat_namespace + endif + else + ifneq ($(OS), Darwin) + ifneq (CYGWIN,$(findstring CYGWIN,$(OS))) + $(1)_LDFLAGS += -Wl,-z,defs + endif + endif + endif + + ifneq ($(OS), Darwin) + $(1)_LDFLAGS += -Wl,-soname=$$($(1)_NAME).$(SO_EXT) + endif + + $(1)_PATH := $$(_d)/$$($(1)_NAME).$(SO_EXT) + + $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ + $$(trace-ld) $(CXX) -o $$(abspath $$@) -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) $$($(1)_LDFLAGS_UNINSTALLED) + + ifneq ($(OS), Darwin) + $(1)_LDFLAGS_USE += -Wl,-rpath,$$(abspath $$(_d)) + endif + $(1)_LDFLAGS_USE += -L$$(_d) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) + + $(1)_INSTALL_PATH := $(DESTDIR)$$($(1)_INSTALL_DIR)/$$($(1)_NAME).$(SO_EXT) + + _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + + $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) + + $$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + $$(trace-ld) $(CXX) -o $$@ -shared $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$($(1)_LDFLAGS_PROPAGATED) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + + $(1)_LDFLAGS_USE_INSTALLED += -L$$(DESTDIR)$$($(1)_INSTALL_DIR) -l$$(patsubst lib%,%,$$(strip $$($(1)_NAME))) + ifneq ($(OS), Darwin) + ifeq ($(SET_RPATH_TO_LIBS), 1) + $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath,$$($(1)_INSTALL_DIR) + else + $(1)_LDFLAGS_USE_INSTALLED += -Wl,-rpath-link,$$($(1)_INSTALL_DIR) + endif + endif + + ifdef $(1)_FORCE_INSTALL + install: $$($(1)_INSTALL_PATH) + endif + + else + + $(1)_PATH := $$(_d)/$$($(1)_NAME).a + + $$($(1)_PATH): $$($(1)_OBJS) | $$(_d)/ + $(trace-ar) $(AR) crs $$@ $$? + + $(1)_LDFLAGS_USE += $$($(1)_PATH) $$($(1)_LDFLAGS) + + $(1)_INSTALL_PATH := $$(libdir)/$$($(1)_NAME).a + + endif + + $(1)_LDFLAGS_USE += $$($(1)_LDFLAGS_PROPAGATED) + $(1)_LDFLAGS_USE_INSTALLED += $$($(1)_LDFLAGS_PROPAGATED) + + # Propagate CFLAGS and CXXFLAGS to the individual object files. + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj)_CFLAGS=$$($(1)_CFLAGS))) + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj)_CXXFLAGS=$$($(1)_CXXFLAGS))) + + # Make each object file depend on the common dependencies. + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj): $$($(1)_COMMON_DEPS) $$(GLOBAL_COMMON_DEPS))) + + # Make each object file have order-only dependencies on the common + # order-only dependencies. This includes the order-only dependencies + # of libraries we're depending on. + $(1)_ORDER_AFTER_CLOSED = $$($(1)_ORDER_AFTER) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_ORDER_AFTER_CLOSED)) + + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj): | $$($(1)_ORDER_AFTER_CLOSED) $$(GLOBAL_ORDER_AFTER))) + + # Include .dep files, if they exist. + $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn))) + -include $$($(1)_DEPS) + + ifndef $(1)_EXCLUDE_FROM_LIBRARY_LIST + libs-list += $$($(1)_PATH) + endif + clean-files += $$(_d)/*.a $$(_d)/*.$(SO_EXT) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) + dist-files += $$(_srcs) +endef diff --git a/third_party/nix/mk/patterns.mk b/third_party/nix/mk/patterns.mk new file mode 100644 index 000000000..3219d9629 --- /dev/null +++ b/third_party/nix/mk/patterns.mk @@ -0,0 +1,11 @@ +$(buildprefix)%.o: %.cc + @mkdir -p "$(dir $@)" + $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + +$(buildprefix)%.o: %.cpp + @mkdir -p "$(dir $@)" + $(trace-cxx) $(CXX) -o $@ -c $< $(GLOBAL_CXXFLAGS) $(GLOBAL_CXXFLAGS_PCH) $(CXXFLAGS) $($@_CXXFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP + +$(buildprefix)%.o: %.c + @mkdir -p "$(dir $@)" + $(trace-cc) $(CC) -o $@ -c $< $(GLOBAL_CFLAGS) $(CFLAGS) $($@_CFLAGS) -MMD -MF $(call filename-to-dep, $@) -MP diff --git a/third_party/nix/mk/programs.mk b/third_party/nix/mk/programs.mk new file mode 100644 index 000000000..d93df4468 --- /dev/null +++ b/third_party/nix/mk/programs.mk @@ -0,0 +1,79 @@ +programs-list := + +# Build a program with symbolic name $(1). The program is defined by +# various variables prefixed by ‘$(1)_’: +# +# - $(1)_DIR: the directory where the (non-installed) program will be +# placed. +# +# - $(1)_SOURCES: the source files of the program. +# +# - $(1)_CFLAGS: additional C compiler flags. +# +# - $(1)_CXXFLAGS: additional C++ compiler flags. +# +# - $(1)_ORDER_AFTER: a set of targets on which the object files of +# this program will have an order-only dependency. +# +# - $(1)_LIBS: the symbolic names of libraries on which this program +# depends. +# +# - $(1)_LDFLAGS: additional linker flags. +# +# - $(1)_INSTALL_DIR: the directory where the program will be +# installed; defaults to $(bindir). +define build-program + _d := $(buildprefix)$$($(1)_DIR) + _srcs := $$(sort $$(foreach src, $$($(1)_SOURCES), $$(src))) + $(1)_OBJS := $$(addprefix $(buildprefix), $$(addsuffix .o, $$(basename $$(_srcs)))) + _libs := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_PATH)) + $(1)_PATH := $$(_d)/$(1) + + $$(eval $$(call create-dir, $$(_d))) + + $$($(1)_PATH): $$($(1)_OBJS) $$(_libs) | $$(_d)/ + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE)) + + $(1)_INSTALL_DIR ?= $$(bindir) + $(1)_INSTALL_PATH := $$($(1)_INSTALL_DIR)/$(1) + + $$(eval $$(call create-dir, $$($(1)_INSTALL_DIR))) + + install: $(DESTDIR)$$($(1)_INSTALL_PATH) + + ifeq ($(BUILD_SHARED_LIBS), 1) + + _libs_final := $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_INSTALL_PATH)) + + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_OBJS) $$(_libs_final) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + $$(trace-ld) $(CXX) -o $$@ $$(LDFLAGS) $$(GLOBAL_LDFLAGS) $$($(1)_OBJS) $$($(1)_LDFLAGS) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_LDFLAGS_USE_INSTALLED)) + + else + + $(DESTDIR)$$($(1)_INSTALL_PATH): $$($(1)_PATH) | $(DESTDIR)$$($(1)_INSTALL_DIR)/ + install -t $(DESTDIR)$$($(1)_INSTALL_DIR) $$< + + endif + + # Propagate CFLAGS and CXXFLAGS to the individual object files. + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj)_CFLAGS=$$($(1)_CFLAGS))) + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj)_CXXFLAGS=$$($(1)_CXXFLAGS))) + + # Make each object file depend on the common dependencies. + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj): $$($(1)_COMMON_DEPS) $$(GLOBAL_COMMON_DEPS))) + + # Make each object file have order-only dependencies on the common + # order-only dependencies. This includes the order-only dependencies + # of libraries we're depending on. + $(1)_ORDER_AFTER_CLOSED = $$($(1)_ORDER_AFTER) $$(foreach lib, $$($(1)_LIBS), $$($$(lib)_ORDER_AFTER_CLOSED)) + + $$(foreach obj, $$($(1)_OBJS), $$(eval $$(obj): | $$($(1)_ORDER_AFTER_CLOSED) $$(GLOBAL_ORDER_AFTER))) + + # Include .dep files, if they exist. + $(1)_DEPS := $$(foreach fn, $$($(1)_OBJS), $$(call filename-to-dep, $$(fn))) + -include $$($(1)_DEPS) + + programs-list += $$($(1)_PATH) + clean-files += $$($(1)_PATH) $$(_d)/*.o $$(_d)/.*.dep $$($(1)_DEPS) $$($(1)_OBJS) + dist-files += $$(_srcs) +endef diff --git a/third_party/nix/mk/templates.mk b/third_party/nix/mk/templates.mk new file mode 100644 index 000000000..c7ac7afbf --- /dev/null +++ b/third_party/nix/mk/templates.mk @@ -0,0 +1,19 @@ +template-files := + +# Create the file $(1) from $(1).in by running config.status (which +# substitutes all ‘@var@’ variables set by the configure script). +define instantiate-template + + clean-files += $(1) + +endef + +ifneq ($(MAKECMDGOALS), clean) + +%.h: %.h.in + $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ + +%: %.in + $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ + +endif diff --git a/third_party/nix/mk/tests.mk b/third_party/nix/mk/tests.mk new file mode 100644 index 000000000..70c30661b --- /dev/null +++ b/third_party/nix/mk/tests.mk @@ -0,0 +1,45 @@ +# Run program $1 as part of ‘make installcheck’. +define run-install-test + + installcheck: $1 + + _installcheck-list += $1 + +endef + +# Color code from https://unix.stackexchange.com/a/10065 +installcheck: + @total=0; failed=0; \ + red=""; \ + green=""; \ + yellow=""; \ + normal=""; \ + if [ -t 1 ]; then \ + red=""; \ + green=""; \ + yellow=""; \ + normal=""; \ + fi; \ + for i in $(_installcheck-list); do \ + total=$$((total + 1)); \ + printf "running test $$i..."; \ + log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \ + status=$$?; \ + if [ $$status -eq 0 ]; then \ + echo " [$${green}PASS$$normal]"; \ + elif [ $$status -eq 99 ]; then \ + echo " [$${yellow}SKIP$$normal]"; \ + else \ + echo " [$${red}FAIL$$normal]"; \ + echo "$$log" | sed 's/^/ /'; \ + failed=$$((failed + 1)); \ + fi; \ + done; \ + if [ "$$failed" != 0 ]; then \ + echo "$${red}$$failed out of $$total tests failed $$normal"; \ + exit 1; \ + else \ + echo "$${green}All tests succeeded$$normal"; \ + fi + +.PHONY: check installcheck diff --git a/third_party/nix/mk/tracing.mk b/third_party/nix/mk/tracing.mk new file mode 100644 index 000000000..13912d3c7 --- /dev/null +++ b/third_party/nix/mk/tracing.mk @@ -0,0 +1,17 @@ +V ?= 0 + +ifeq ($(V), 0) + + trace-gen = @echo " GEN " $@; + trace-cc = @echo " CC " $@; + trace-cxx = @echo " CXX " $@; + trace-ld = @echo " LD " $@; + trace-ar = @echo " AR " $@; + trace-install = @echo " INST " $@; + trace-javac = @echo " JAVAC " $@; + trace-jar = @echo " JAR " $@; + trace-mkdir = @echo " MKDIR " $@; + + suppress = @ + +endif diff --git a/third_party/nix/nix.spec.in b/third_party/nix/nix.spec.in new file mode 100644 index 000000000..6b9e37637 --- /dev/null +++ b/third_party/nix/nix.spec.in @@ -0,0 +1,173 @@ +%undefine _hardened_build + +%global nixbld_user "nix-builder-" +%global nixbld_group "nixbld" + +# NOTE: BUILD on EL7 requires +# - Centos / RHEL7 software collection repository +# yum install centos-release-scl +# +# - Recent boost backport +# curl https://copr.fedorainfracloud.org/coprs/whosthere/boost/repo/epel-7/whosthere-boost-epel-7.repo -o /etc/yum.repos.d/whosthere-boost-epel-7.repo +# + +# Disable documentation generation +# necessary on some platforms +%bcond_without docgen + +Summary: The Nix software deployment system +Name: nix +Version: @PACKAGE_VERSION@ +Release: 2%{?dist} +License: LGPLv2+ +Group: Applications/System +URL: http://nixos.org/ +Source0: %{name}-%{version}.tar.bz2 + +Requires: curl +Requires: bzip2 +Requires: gzip +Requires: xz +BuildRequires: bison +BuildRequires: boost-devel >= 1.60 +BuildRequires: bzip2-devel + +# for RHEL <= 7, we need software collections for a C++14 compatible compatible compiler +%if 0%{?rhel} +BuildRequires: devtoolset-7-gcc +BuildRequires: devtoolset-7-gcc-c++ +%endif + +BuildRequires: flex +BuildRequires: libcurl-devel +BuildRequires: libseccomp-devel +BuildRequires: openssl-devel +BuildRequires: sqlite-devel +BuildRequires: xz-devel + +%description +Nix is a purely functional package manager. It allows multiple +versions of a package to be installed side-by-side, ensures that +dependency specifications are complete, supports atomic upgrades and +rollbacks, allows non-root users to install software, and has many +other features. It is the basis of the NixOS Linux distribution, but +it can be used equally well under other Unix systems. + +%package devel +Summary: Development files for %{name} +Requires: %{name}%{?_isa} = %{version}-%{release} + +%description devel +The %{name}-devel package contains libraries and header files for +developing applications that use %{name}. + + +%package doc +Summary: Documentation files for %{name} +BuildArch: noarch +Requires: %{name} = %{version}-%{release} + +%description doc +The %{name}-doc package contains documentation files for %{name}. + +%prep +%setup -q + + +%build +%if 0%{?rhel} +source /opt/rh/devtoolset-7/enable +%endif +extraFlags= +# - override docdir so large documentation files are owned by the +# -doc subpackage +# - set localstatedir by hand to the preferred nix value +%configure --localstatedir=/nix/var \ + %{!?without_docgen:--disable-doc-gen} \ + --docdir=%{_defaultdocdir}/%{name}-doc-%{version} \ + $extraFlags +make V=1 %{?_smp_mflags} + + +%install +%if 0%{?rhel} +source /opt/rh/devtoolset-7/enable +%endif + +make DESTDIR=$RPM_BUILD_ROOT install + +find $RPM_BUILD_ROOT -name '*.la' -exec rm -f {} ';' + +# make the store +mkdir -p $RPM_BUILD_ROOT/nix/store +chmod 1775 $RPM_BUILD_ROOT/nix/store + +# make per-user directories +for d in profiles gcroots; +do + mkdir -p $RPM_BUILD_ROOT/nix/var/nix/$d/per-user + chmod 755 $RPM_BUILD_ROOT/nix/var/nix/$d/per-user +done + +# fix permission of nix profile +# (until this is fixed in the relevant Makefile) +chmod -x $RPM_BUILD_ROOT%{_sysconfdir}/profile.d/nix.sh + +# we ship this file in the base package +rm -f $RPM_BUILD_ROOT%{_defaultdocdir}/%{name}-doc-%{version}/README + +# Get rid of Upstart job. +rm -rf $RPM_BUILD_ROOT%{_sysconfdir}/init + + +%clean +rm -rf $RPM_BUILD_ROOT + + +%pre +getent group %{nixbld_group} >/dev/null || groupadd -r %{nixbld_group} +for i in $(seq 10); +do + getent passwd %{nixbld_user}$i >/dev/null || \ + useradd -r -g %{nixbld_group} -G %{nixbld_group} -d /var/empty \ + -s %{_sbindir}/nologin \ + -c "Nix build user $i" %{nixbld_user}$i +done + +%post +chgrp %{nixbld_group} /nix/store +%if ! 0%{?rhel} || 0%{?rhel} >= 7 +# Enable and start Nix worker +systemctl enable nix-daemon.socket nix-daemon.service +systemctl start nix-daemon.socket +%endif + +%files +%license COPYING +%{_bindir}/nix* +%{_libdir}/*.so +%{_prefix}/libexec/* +%if ! 0%{?rhel} || 0%{?rhel} >= 7 +%{_prefix}/lib/systemd/system/nix-daemon.socket +%{_prefix}/lib/systemd/system/nix-daemon.service +%endif +%{_datadir}/nix +#%if ! %{without docgen} +#%{_mandir}/man1/*.1* +#%{_mandir}/man5/*.5* +#%{_mandir}/man8/*.8* +#%endif +%config(noreplace) %{_sysconfdir}/profile.d/nix.sh +%config(noreplace) %{_sysconfdir}/profile.d/nix-daemon.sh +/nix + +%files devel +%{_includedir}/nix +%{_prefix}/lib/pkgconfig/*.pc + + +#%if ! %{without docgen} +#%files doc +#%docdir %{_defaultdocdir}/%{name}-doc-%{version} +#%{_defaultdocdir}/%{name}-doc-%{version} +#%endif diff --git a/third_party/nix/perl/MANIFEST b/third_party/nix/perl/MANIFEST new file mode 100644 index 000000000..08897647c --- /dev/null +++ b/third_party/nix/perl/MANIFEST @@ -0,0 +1,7 @@ +Changes +Makefile.PL +MANIFEST +Nix.xs +README +t/Nix.t +lib/Nix.pm diff --git a/third_party/nix/perl/Makefile b/third_party/nix/perl/Makefile new file mode 100644 index 000000000..f36f5d0e9 --- /dev/null +++ b/third_party/nix/perl/Makefile @@ -0,0 +1,7 @@ +makefiles = local.mk + +GLOBAL_CXXFLAGS += -g -Wall + +-include Makefile.config + +include mk/lib.mk diff --git a/third_party/nix/perl/Makefile.config.in b/third_party/nix/perl/Makefile.config.in new file mode 100644 index 000000000..c87d4817e --- /dev/null +++ b/third_party/nix/perl/Makefile.config.in @@ -0,0 +1,18 @@ +CC = @CC@ +CFLAGS = @CFLAGS@ +CXX = @CXX@ +CXXFLAGS = @CXXFLAGS@ +HAVE_SODIUM = @HAVE_SODIUM@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +SODIUM_LIBS = @SODIUM_LIBS@ +NIX_CFLAGS = @NIX_CFLAGS@ +NIX_LIBS = @NIX_LIBS@ +nixbindir = @nixbindir@ +curl = @curl@ +nixlibexecdir = @nixlibexecdir@ +nixlocalstatedir = @nixlocalstatedir@ +perl = @perl@ +perllibdir = @perllibdir@ +nixstoredir = @nixstoredir@ +nixsysconfdir = @nixsysconfdir@ diff --git a/third_party/nix/perl/configure.ac b/third_party/nix/perl/configure.ac new file mode 100644 index 000000000..e8e3610a8 --- /dev/null +++ b/third_party/nix/perl/configure.ac @@ -0,0 +1,101 @@ +AC_INIT(nix-perl, m4_esyscmd([bash -c "echo -n $(cat ../.version)$VERSION_SUFFIX"])) +AC_CONFIG_SRCDIR(MANIFEST) +AC_CONFIG_AUX_DIR(../config) + +# Set default flags for nix (as per AC_PROG_CC/CXX docs), +# while still allowing the user to override them from the command line. +: ${CFLAGS="-O3"} +: ${CXXFLAGS="-O3"} +AC_PROG_CC +AC_PROG_CXX +AX_CXX_COMPILE_STDCXX_11 + +# Use 64-bit file system calls so that we can support files > 2 GiB. +AC_SYS_LARGEFILE + +AC_DEFUN([NEED_PROG], +[ +AC_PATH_PROG($1, $2) +if test -z "$$1"; then + AC_MSG_ERROR([$2 is required]) +fi +]) + +NEED_PROG(perl, perl) +NEED_PROG(curl, curl) +NEED_PROG(bzip2, bzip2) +NEED_PROG(xz, xz) + +# Test that Perl has the open/fork feature (Perl 5.8.0 and beyond). +AC_MSG_CHECKING([whether Perl is recent enough]) +if ! $perl -e 'open(FOO, "-|", "true"); while () { print; }; close FOO or die;'; then + AC_MSG_RESULT(no) + AC_MSG_ERROR([Your Perl version is too old. Nix requires Perl 5.8.0 or newer.]) +fi +AC_MSG_RESULT(yes) + + +# Figure out where to install Perl modules. +AC_MSG_CHECKING([for the Perl installation prefix]) +perlversion=$($perl -e 'use Config; print $Config{version};') +perlarchname=$($perl -e 'use Config; print $Config{archname};') +AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) +AC_MSG_RESULT($perllibdir) + +# Look for libsodium, an optional dependency. +PKG_CHECK_MODULES([SODIUM], [libsodium], + [AC_DEFINE([HAVE_SODIUM], [1], [Whether to use libsodium for cryptography.]) + CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS" + have_sodium=1], [have_sodium=]) +AC_SUBST(HAVE_SODIUM, [$have_sodium]) + +# Check for the required Perl dependencies (DBI and DBD::SQLite). +perlFlags="-I$perllibdir" + +AC_ARG_WITH(dbi, AC_HELP_STRING([--with-dbi=PATH], + [prefix of the Perl DBI library]), + perlFlags="$perlFlags -I$withval") + +AC_ARG_WITH(dbd-sqlite, AC_HELP_STRING([--with-dbd-sqlite=PATH], + [prefix of the Perl DBD::SQLite library]), + perlFlags="$perlFlags -I$withval") + +AC_MSG_CHECKING([whether DBD::SQLite works]) +if ! $perl $perlFlags -e 'use DBI; use DBD::SQLite;' 2>&5; then + AC_MSG_RESULT(no) + AC_MSG_FAILURE([The Perl modules DBI and/or DBD::SQLite are missing.]) +fi +AC_MSG_RESULT(yes) + +AC_SUBST(perlFlags) + +PKG_CHECK_MODULES([NIX], [nix-store]) + +NEED_PROG([NIX_INSTANTIATE_PROGRAM], [nix-instantiate]) + +# Get nix configure values +nixbindir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixBinDir | tr -d \") +nixlibexecdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixLibexecDir | tr -d \") +nixlocalstatedir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixLocalstateDir | tr -d \") +nixsysconfdir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixSysconfDir | tr -d \") +nixstoredir=$("$NIX_INSTANTIATE_PROGRAM" --eval '' -A nixStoreDir | tr -d \") +AC_SUBST(nixbindir) +AC_SUBST(nixlibexecdir) +AC_SUBST(nixlocalstatedir) +AC_SUBST(nixsysconfdir) +AC_SUBST(nixstoredir) + +# Expand all variables in config.status. +test "$prefix" = NONE && prefix=$ac_default_prefix +test "$exec_prefix" = NONE && exec_prefix='${prefix}' +for name in $ac_subst_vars; do + declare $name="$(eval echo "${!name}")" + declare $name="$(eval echo "${!name}")" + declare $name="$(eval echo "${!name}")" +done + +rm -f Makefile.config +ln -sfn ../mk mk + +AC_CONFIG_FILES([]) +AC_OUTPUT diff --git a/third_party/nix/perl/lib/Nix/Config.pm.in b/third_party/nix/perl/lib/Nix/Config.pm.in new file mode 100644 index 000000000..67a20c3f4 --- /dev/null +++ b/third_party/nix/perl/lib/Nix/Config.pm.in @@ -0,0 +1,34 @@ +package Nix::Config; + +use MIME::Base64; + +$version = "@PACKAGE_VERSION@"; + +$binDir = $ENV{"NIX_BIN_DIR"} || "@nixbindir@"; +$libexecDir = $ENV{"NIX_LIBEXEC_DIR"} || "@nixlibexecdir@"; +$stateDir = $ENV{"NIX_STATE_DIR"} || "@nixlocalstatedir@/nix"; +$logDir = $ENV{"NIX_LOG_DIR"} || "@nixlocalstatedir@/log/nix"; +$confDir = $ENV{"NIX_CONF_DIR"} || "@nixsysconfdir@/nix"; +$storeDir = $ENV{"NIX_STORE_DIR"} || "@nixstoredir@"; + +$bzip2 = "@bzip2@"; +$xz = "@xz@"; +$curl = "@curl@"; + +$useBindings = 1; + +%config = (); + +sub readConfig { + my $config = "$confDir/nix.conf"; + return unless -f $config; + + open CONFIG, "<$config" or die "cannot open '$config'"; + while () { + /^\s*([\w\-\.]+)\s*=\s*(.*)$/ or next; + $config{$1} = $2; + } + close CONFIG; +} + +return 1; diff --git a/third_party/nix/perl/lib/Nix/CopyClosure.pm b/third_party/nix/perl/lib/Nix/CopyClosure.pm new file mode 100644 index 000000000..902ee1a1b --- /dev/null +++ b/third_party/nix/perl/lib/Nix/CopyClosure.pm @@ -0,0 +1,61 @@ +package Nix::CopyClosure; + +use utf8; +use strict; +use Nix::Config; +use Nix::Store; +use Nix::SSH; +use List::Util qw(sum); +use IPC::Open2; + + +sub copyToOpen { + my ($from, $to, $sshHost, $storePaths, $includeOutputs, $dryRun, $useSubstitutes) = @_; + + $useSubstitutes = 0 if $dryRun || !defined $useSubstitutes; + + # Get the closure of this path. + my @closure = reverse(topoSortPaths(computeFSClosure(0, $includeOutputs, + map { followLinksToStorePath $_ } @{$storePaths}))); + + # Send the "query valid paths" command with the "lock" option + # enabled. This prevents a race where the remote host + # garbage-collect paths that are already there. Optionally, ask + # the remote host to substitute missing paths. + syswrite($to, pack("L{url} eq $info->{url}; + } + + push @{$narFileList}, $info if !$found; +} + + +sub addPatch { + my ($patches, $storePath, $patch) = @_; + + $$patches{$storePath} = [] + unless defined $$patches{$storePath}; + + my $patchList = $$patches{$storePath}; + + my $found = 0; + foreach my $patch2 (@{$patchList}) { + $found = 1 if + $patch2->{url} eq $patch->{url} && + $patch2->{basePath} eq $patch->{basePath}; + } + + push @{$patchList}, $patch if !$found; + + return !$found; +} + + +sub readManifest_ { + my ($manifest, $addNAR, $addPatch) = @_; + + # Decompress the manifest if necessary. + if ($manifest =~ /\.bz2$/) { + open MANIFEST, "$Nix::Config::bzip2 -d < $manifest |" + or die "cannot decompress '$manifest': $!"; + } else { + open MANIFEST, "<$manifest" + or die "cannot open '$manifest': $!"; + } + + my $inside = 0; + my $type; + + my $manifestVersion = 2; + + my ($storePath, $url, $hash, $size, $basePath, $baseHash, $patchType); + my ($narHash, $narSize, $references, $deriver, $copyFrom, $system, $compressionType); + + while () { + chomp; + s/\#.*$//g; + next if (/^$/); + + if (!$inside) { + + if (/^\s*(\w*)\s*\{$/) { + $type = $1; + $type = "narfile" if $type eq ""; + $inside = 1; + undef $storePath; + undef $url; + undef $hash; + undef $size; + undef $narHash; + undef $narSize; + undef $basePath; + undef $baseHash; + undef $patchType; + undef $system; + $references = ""; + $deriver = ""; + $compressionType = "bzip2"; + } + + } else { + + if (/^\}$/) { + $inside = 0; + + if ($type eq "narfile") { + &$addNAR($storePath, + { url => $url, hash => $hash, size => $size + , narHash => $narHash, narSize => $narSize + , references => $references + , deriver => $deriver + , system => $system + , compressionType => $compressionType + }); + } + + elsif ($type eq "patch") { + &$addPatch($storePath, + { url => $url, hash => $hash, size => $size + , basePath => $basePath, baseHash => $baseHash + , narHash => $narHash, narSize => $narSize + , patchType => $patchType + }); + } + + } + + elsif (/^\s*StorePath:\s*(\/\S+)\s*$/) { $storePath = $1; } + elsif (/^\s*CopyFrom:\s*(\/\S+)\s*$/) { $copyFrom = $1; } + elsif (/^\s*Hash:\s*(\S+)\s*$/) { $hash = $1; } + elsif (/^\s*URL:\s*(\S+)\s*$/) { $url = $1; } + elsif (/^\s*Compression:\s*(\S+)\s*$/) { $compressionType = $1; } + elsif (/^\s*Size:\s*(\d+)\s*$/) { $size = $1; } + elsif (/^\s*BasePath:\s*(\/\S+)\s*$/) { $basePath = $1; } + elsif (/^\s*BaseHash:\s*(\S+)\s*$/) { $baseHash = $1; } + elsif (/^\s*Type:\s*(\S+)\s*$/) { $patchType = $1; } + elsif (/^\s*NarHash:\s*(\S+)\s*$/) { $narHash = $1; } + elsif (/^\s*NarSize:\s*(\d+)\s*$/) { $narSize = $1; } + elsif (/^\s*References:\s*(.*)\s*$/) { $references = $1; } + elsif (/^\s*Deriver:\s*(\S+)\s*$/) { $deriver = $1; } + elsif (/^\s*ManifestVersion:\s*(\d+)\s*$/) { $manifestVersion = $1; } + elsif (/^\s*System:\s*(\S+)\s*$/) { $system = $1; } + + # Compatibility; + elsif (/^\s*NarURL:\s*(\S+)\s*$/) { $url = $1; } + elsif (/^\s*MD5:\s*(\S+)\s*$/) { $hash = "md5:$1"; } + + } + } + + close MANIFEST; + + return $manifestVersion; +} + + +sub readManifest { + my ($manifest, $narFiles, $patches) = @_; + readManifest_($manifest, + sub { addNAR($narFiles, @_); }, + sub { addPatch($patches, @_); } ); +} + + +sub writeManifest { + my ($manifest, $narFiles, $patches, $noCompress) = @_; + + open MANIFEST, ">$manifest.tmp"; # !!! check exclusive + + print MANIFEST "version {\n"; + print MANIFEST " ManifestVersion: 3\n"; + print MANIFEST "}\n"; + + foreach my $storePath (sort (keys %{$narFiles})) { + my $narFileList = $$narFiles{$storePath}; + foreach my $narFile (@{$narFileList}) { + print MANIFEST "{\n"; + print MANIFEST " StorePath: $storePath\n"; + print MANIFEST " NarURL: $narFile->{url}\n"; + print MANIFEST " Compression: $narFile->{compressionType}\n"; + print MANIFEST " Hash: $narFile->{hash}\n" if defined $narFile->{hash}; + print MANIFEST " Size: $narFile->{size}\n" if defined $narFile->{size}; + print MANIFEST " NarHash: $narFile->{narHash}\n"; + print MANIFEST " NarSize: $narFile->{narSize}\n" if $narFile->{narSize}; + print MANIFEST " References: $narFile->{references}\n" + if defined $narFile->{references} && $narFile->{references} ne ""; + print MANIFEST " Deriver: $narFile->{deriver}\n" + if defined $narFile->{deriver} && $narFile->{deriver} ne ""; + print MANIFEST " System: $narFile->{system}\n" if defined $narFile->{system}; + print MANIFEST "}\n"; + } + } + + foreach my $storePath (sort (keys %{$patches})) { + my $patchList = $$patches{$storePath}; + foreach my $patch (@{$patchList}) { + print MANIFEST "patch {\n"; + print MANIFEST " StorePath: $storePath\n"; + print MANIFEST " NarURL: $patch->{url}\n"; + print MANIFEST " Hash: $patch->{hash}\n"; + print MANIFEST " Size: $patch->{size}\n"; + print MANIFEST " NarHash: $patch->{narHash}\n"; + print MANIFEST " NarSize: $patch->{narSize}\n" if $patch->{narSize}; + print MANIFEST " BasePath: $patch->{basePath}\n"; + print MANIFEST " BaseHash: $patch->{baseHash}\n"; + print MANIFEST " Type: $patch->{patchType}\n"; + print MANIFEST "}\n"; + } + } + + + close MANIFEST; + + rename("$manifest.tmp", $manifest) + or die "cannot rename $manifest.tmp: $!"; + + + # Create a bzipped manifest. + unless (defined $noCompress) { + system("$Nix::Config::bzip2 < $manifest > $manifest.bz2.tmp") == 0 + or die "cannot compress manifest"; + + rename("$manifest.bz2.tmp", "$manifest.bz2") + or die "cannot rename $manifest.bz2.tmp: $!"; + } +} + + +# Return a fingerprint of a store path to be used in binary cache +# signatures. It contains the store path, the base-32 SHA-256 hash of +# the contents of the path, and the references. +sub fingerprintPath { + my ($storePath, $narHash, $narSize, $references) = @_; + die if substr($storePath, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; + die if substr($narHash, 0, 7) ne "sha256:"; + # Convert hash from base-16 to base-32, if necessary. + $narHash = "sha256:" . convertHash("sha256", substr($narHash, 7), 1) + if length($narHash) == 71; + die if length($narHash) != 59; + foreach my $ref (@{$references}) { + die if substr($ref, 0, length($Nix::Config::storeDir)) ne $Nix::Config::storeDir; + } + return "1;" . $storePath . ";" . $narHash . ";" . $narSize . ";" . join(",", @{$references}); +} + + +# Parse a NAR info file. +sub parseNARInfo { + my ($storePath, $content, $requireValidSig, $location) = @_; + + my ($storePath2, $url, $fileHash, $fileSize, $narHash, $narSize, $deriver, $system, $sig); + my $compression = "bzip2"; + my @refs; + + foreach my $line (split "\n", $content) { + return undef unless $line =~ /^(.*): (.*)$/; + if ($1 eq "StorePath") { $storePath2 = $2; } + elsif ($1 eq "URL") { $url = $2; } + elsif ($1 eq "Compression") { $compression = $2; } + elsif ($1 eq "FileHash") { $fileHash = $2; } + elsif ($1 eq "FileSize") { $fileSize = int($2); } + elsif ($1 eq "NarHash") { $narHash = $2; } + elsif ($1 eq "NarSize") { $narSize = int($2); } + elsif ($1 eq "References") { @refs = split / /, $2; } + elsif ($1 eq "Deriver") { $deriver = $2; } + elsif ($1 eq "System") { $system = $2; } + elsif ($1 eq "Sig") { $sig = $2; } + } + + return undef if $storePath ne $storePath2 || !defined $url || !defined $narHash; + + my $res = + { url => $url + , compression => $compression + , fileHash => $fileHash + , fileSize => $fileSize + , narHash => $narHash + , narSize => $narSize + , refs => [ @refs ] + , deriver => $deriver + , system => $system + }; + + if ($requireValidSig) { + # FIXME: might be useful to support multiple signatures per .narinfo. + + if (!defined $sig) { + warn "NAR info file '$location' lacks a signature; ignoring\n"; + return undef; + } + my ($keyName, $sig64) = split ":", $sig; + return undef unless defined $keyName && defined $sig64; + + my $publicKey = $Nix::Config::binaryCachePublicKeys{$keyName}; + if (!defined $publicKey) { + warn "NAR info file '$location' is signed by unknown key '$keyName'; ignoring\n"; + return undef; + } + + my $fingerprint; + eval { + $fingerprint = fingerprintPath( + $storePath, $narHash, $narSize, + [ map { "$Nix::Config::storeDir/$_" } @refs ]); + }; + if ($@) { + warn "cannot compute fingerprint of '$location'; ignoring\n"; + return undef; + } + + if (!checkSignature($publicKey, decode_base64($sig64), $fingerprint)) { + warn "NAR info file '$location' has an incorrect signature; ignoring\n"; + return undef; + } + + $res->{signedBy} = $keyName; + } + + return $res; +} + + +return 1; diff --git a/third_party/nix/perl/lib/Nix/SSH.pm b/third_party/nix/perl/lib/Nix/SSH.pm new file mode 100644 index 000000000..490ba0ea9 --- /dev/null +++ b/third_party/nix/perl/lib/Nix/SSH.pm @@ -0,0 +1,110 @@ +package Nix::SSH; + +use utf8; +use strict; +use File::Temp qw(tempdir); +use IPC::Open2; + +our @ISA = qw(Exporter); +our @EXPORT = qw( + @globalSshOpts + readN readInt readString readStrings + writeInt writeString writeStrings + connectToRemoteNix +); + + +our @globalSshOpts = split ' ', ($ENV{"NIX_SSHOPTS"} or ""); + + +sub readN { + my ($bytes, $from) = @_; + my $res = ""; + while ($bytes > 0) { + my $s; + my $n = sysread($from, $s, $bytes); + die "I/O error reading from remote side\n" if !defined $n; + die "got EOF while expecting $bytes bytes from remote side\n" if !$n; + $bytes -= $n; + $res .= $s; + } + return $res; +} + + +sub readInt { + my ($from) = @_; + return unpack("L= 0x300; + + return ($from, $to, $pid); +} + + +1; diff --git a/third_party/nix/perl/lib/Nix/Store.pm b/third_party/nix/perl/lib/Nix/Store.pm new file mode 100644 index 000000000..d226264d4 --- /dev/null +++ b/third_party/nix/perl/lib/Nix/Store.pm @@ -0,0 +1,95 @@ +package Nix::Store; + +use strict; +use warnings; +use Nix::Config; + +require Exporter; + +our @ISA = qw(Exporter); + +our %EXPORT_TAGS = ( 'all' => [ qw( ) ] ); + +our @EXPORT_OK = ( @{ $EXPORT_TAGS{'all'} } ); + +our @EXPORT = qw( + setVerbosity + isValidPath queryReferences queryPathInfo queryDeriver queryPathHash + queryPathFromHashPart + topoSortPaths computeFSClosure followLinksToStorePath exportPaths importPaths + hashPath hashFile hashString convertHash + signString checkSignature + addToStore makeFixedOutputPath + derivationFromPath + addTempRoot +); + +our $VERSION = '0.15'; + +sub backtick { + open(RES, "-|", @_) or die; + local $/; + my $res = || ""; + close RES or die; + return $res; +} + +if ($Nix::Config::useBindings) { + require XSLoader; + XSLoader::load('Nix::Store', $VERSION); +} else { + + # Provide slow fallbacks of some functions on platforms that don't + # support the Perl bindings. + + use File::Temp; + use Fcntl qw/F_SETFD/; + + *hashFile = sub { + my ($algo, $base32, $path) = @_; + my $res = backtick("$Nix::Config::binDir/nix-hash", "--flat", $path, "--type", $algo, $base32 ? "--base32" : ()); + chomp $res; + return $res; + }; + + *hashPath = sub { + my ($algo, $base32, $path) = @_; + my $res = backtick("$Nix::Config::binDir/nix-hash", $path, "--type", $algo, $base32 ? "--base32" : ()); + chomp $res; + return $res; + }; + + *hashString = sub { + my ($algo, $base32, $s) = @_; + my $fh = File::Temp->new(); + print $fh $s; + my $res = backtick("$Nix::Config::binDir/nix-hash", $fh->filename, "--type", $algo, $base32 ? "--base32" : ()); + chomp $res; + return $res; + }; + + *addToStore = sub { + my ($srcPath, $recursive, $algo) = @_; + die "not implemented" if $recursive || $algo ne "sha256"; + my $res = backtick("$Nix::Config::binDir/nix-store", "--add", $srcPath); + chomp $res; + return $res; + }; + + *isValidPath = sub { + my ($path) = @_; + my $res = backtick("$Nix::Config::binDir/nix-store", "--check-validity", "--print-invalid", $path); + chomp $res; + return $res ne $path; + }; + + *queryPathHash = sub { + my ($path) = @_; + my $res = backtick("$Nix::Config::binDir/nix-store", "--query", "--hash", $path); + chomp $res; + return $res; + }; +} + +1; +__END__ diff --git a/third_party/nix/perl/lib/Nix/Store.xs b/third_party/nix/perl/lib/Nix/Store.xs new file mode 100644 index 000000000..ce553bb53 --- /dev/null +++ b/third_party/nix/perl/lib/Nix/Store.xs @@ -0,0 +1,346 @@ +#include "config.h" + +#include "EXTERN.h" +#include "perl.h" +#include "XSUB.h" + +/* Prevent a clash between some Perl and libstdc++ macros. */ +#undef do_open +#undef do_close + +#include "derivations.hh" +#include "globals.hh" +#include "store-api.hh" +#include "util.hh" +#include "crypto.hh" + +#if HAVE_SODIUM +#include +#endif + + +using namespace nix; + + +static ref store() +{ + static std::shared_ptr _store; + if (!_store) { + try { + loadConfFile(); + settings.lockCPU = false; + _store = openStore(); + } catch (Error & e) { + croak("%s", e.what()); + } + } + return ref(_store); +} + + +MODULE = Nix::Store PACKAGE = Nix::Store +PROTOTYPES: ENABLE + + +#undef dNOOP // Hack to work around "error: declaration of 'Perl___notused' has a different language linkage" error message on clang. +#define dNOOP + + +void init() + CODE: + store(); + + +void setVerbosity(int level) + CODE: + verbosity = (Verbosity) level; + + +int isValidPath(char * path) + CODE: + try { + RETVAL = store()->isValidPath(path); + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + + +SV * queryReferences(char * path) + PPCODE: + try { + PathSet paths = store()->queryPathInfo(path)->references; + for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) + XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * queryPathHash(char * path) + PPCODE: + try { + auto s = store()->queryPathInfo(path)->narHash.to_string(); + XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * queryDeriver(char * path) + PPCODE: + try { + auto deriver = store()->queryPathInfo(path)->deriver; + if (deriver == "") XSRETURN_UNDEF; + XPUSHs(sv_2mortal(newSVpv(deriver.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * queryPathInfo(char * path, int base32) + PPCODE: + try { + auto info = store()->queryPathInfo(path); + if (info->deriver == "") + XPUSHs(&PL_sv_undef); + else + XPUSHs(sv_2mortal(newSVpv(info->deriver.c_str(), 0))); + auto s = info->narHash.to_string(base32 ? Base32 : Base16); + XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); + mXPUSHi(info->registrationTime); + mXPUSHi(info->narSize); + AV * arr = newAV(); + for (PathSet::iterator i = info->references.begin(); i != info->references.end(); ++i) + av_push(arr, newSVpv(i->c_str(), 0)); + XPUSHs(sv_2mortal(newRV((SV *) arr))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * queryPathFromHashPart(char * hashPart) + PPCODE: + try { + Path path = store()->queryPathFromHashPart(hashPart); + XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * computeFSClosure(int flipDirection, int includeOutputs, ...) + PPCODE: + try { + PathSet paths; + for (int n = 2; n < items; ++n) + store()->computeFSClosure(SvPV_nolen(ST(n)), paths, flipDirection, includeOutputs); + for (PathSet::iterator i = paths.begin(); i != paths.end(); ++i) + XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * topoSortPaths(...) + PPCODE: + try { + PathSet paths; + for (int n = 0; n < items; ++n) paths.insert(SvPV_nolen(ST(n))); + Paths sorted = store()->topoSortPaths(paths); + for (Paths::iterator i = sorted.begin(); i != sorted.end(); ++i) + XPUSHs(sv_2mortal(newSVpv(i->c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * followLinksToStorePath(char * path) + CODE: + try { + RETVAL = newSVpv(store()->followLinksToStorePath(path).c_str(), 0); + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + + +void exportPaths(int fd, ...) + PPCODE: + try { + Paths paths; + for (int n = 1; n < items; ++n) paths.push_back(SvPV_nolen(ST(n))); + FdSink sink(fd); + store()->exportPaths(paths, sink); + } catch (Error & e) { + croak("%s", e.what()); + } + + +void importPaths(int fd, int dontCheckSigs) + PPCODE: + try { + FdSource source(fd); + store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * hashPath(char * algo, int base32, char * path) + PPCODE: + try { + Hash h = hashPath(parseHashType(algo), path).first; + auto s = h.to_string(base32 ? Base32 : Base16, false); + XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * hashFile(char * algo, int base32, char * path) + PPCODE: + try { + Hash h = hashFile(parseHashType(algo), path); + auto s = h.to_string(base32 ? Base32 : Base16, false); + XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * hashString(char * algo, int base32, char * s) + PPCODE: + try { + Hash h = hashString(parseHashType(algo), s); + auto s = h.to_string(base32 ? Base32 : Base16, false); + XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * convertHash(char * algo, char * s, int toBase32) + PPCODE: + try { + Hash h(s, parseHashType(algo)); + string s = h.to_string(toBase32 ? Base32 : Base16, false); + XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * signString(char * secretKey_, char * msg) + PPCODE: + try { +#if HAVE_SODIUM + auto sig = SecretKey(secretKey_).signDetached(msg); + XPUSHs(sv_2mortal(newSVpv(sig.c_str(), sig.size()))); +#else + throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +#endif + } catch (Error & e) { + croak("%s", e.what()); + } + + +int checkSignature(SV * publicKey_, SV * sig_, char * msg) + CODE: + try { +#if HAVE_SODIUM + STRLEN publicKeyLen; + unsigned char * publicKey = (unsigned char *) SvPV(publicKey_, publicKeyLen); + if (publicKeyLen != crypto_sign_PUBLICKEYBYTES) + throw Error("public key is not valid"); + + STRLEN sigLen; + unsigned char * sig = (unsigned char *) SvPV(sig_, sigLen); + if (sigLen != crypto_sign_BYTES) + throw Error("signature is not valid"); + + RETVAL = crypto_sign_verify_detached(sig, (unsigned char *) msg, strlen(msg), publicKey) == 0; +#else + throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +#endif + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + + +SV * addToStore(char * srcPath, int recursive, char * algo) + PPCODE: + try { + Path path = store()->addToStore(baseNameOf(srcPath), srcPath, recursive, parseHashType(algo)); + XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) + PPCODE: + try { + Hash h(hash, parseHashType(algo)); + Path path = store()->makeFixedOutputPath(recursive, h, name); + XPUSHs(sv_2mortal(newSVpv(path.c_str(), 0))); + } catch (Error & e) { + croak("%s", e.what()); + } + + +SV * derivationFromPath(char * drvPath) + PREINIT: + HV *hash; + CODE: + try { + Derivation drv = store()->derivationFromPath(drvPath); + hash = newHV(); + + HV * outputs = newHV(); + for (DerivationOutputs::iterator i = drv.outputs.begin(); i != drv.outputs.end(); ++i) + hv_store(outputs, i->first.c_str(), i->first.size(), newSVpv(i->second.path.c_str(), 0), 0); + hv_stores(hash, "outputs", newRV((SV *) outputs)); + + AV * inputDrvs = newAV(); + for (DerivationInputs::iterator i = drv.inputDrvs.begin(); i != drv.inputDrvs.end(); ++i) + av_push(inputDrvs, newSVpv(i->first.c_str(), 0)); // !!! ignores i->second + hv_stores(hash, "inputDrvs", newRV((SV *) inputDrvs)); + + AV * inputSrcs = newAV(); + for (PathSet::iterator i = drv.inputSrcs.begin(); i != drv.inputSrcs.end(); ++i) + av_push(inputSrcs, newSVpv(i->c_str(), 0)); + hv_stores(hash, "inputSrcs", newRV((SV *) inputSrcs)); + + hv_stores(hash, "platform", newSVpv(drv.platform.c_str(), 0)); + hv_stores(hash, "builder", newSVpv(drv.builder.c_str(), 0)); + + AV * args = newAV(); + for (Strings::iterator i = drv.args.begin(); i != drv.args.end(); ++i) + av_push(args, newSVpv(i->c_str(), 0)); + hv_stores(hash, "args", newRV((SV *) args)); + + HV * env = newHV(); + for (StringPairs::iterator i = drv.env.begin(); i != drv.env.end(); ++i) + hv_store(env, i->first.c_str(), i->first.size(), newSVpv(i->second.c_str(), 0), 0); + hv_stores(hash, "env", newRV((SV *) env)); + + RETVAL = newRV_noinc((SV *)hash); + } catch (Error & e) { + croak("%s", e.what()); + } + OUTPUT: + RETVAL + + +void addTempRoot(char * storePath) + PPCODE: + try { + store()->addTempRoot(storePath); + } catch (Error & e) { + croak("%s", e.what()); + } diff --git a/third_party/nix/perl/lib/Nix/Utils.pm b/third_party/nix/perl/lib/Nix/Utils.pm new file mode 100644 index 000000000..44955a706 --- /dev/null +++ b/third_party/nix/perl/lib/Nix/Utils.pm @@ -0,0 +1,47 @@ +package Nix::Utils; + +use utf8; +use File::Temp qw(tempdir); + +our @ISA = qw(Exporter); +our @EXPORT = qw(checkURL uniq writeFile readFile mkTempDir); + +$urlRE = "(?: [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*]+ )"; + +sub checkURL { + my ($url) = @_; + die "invalid URL '$url'\n" unless $url =~ /^ $urlRE $ /x; +} + +sub uniq { + my %seen; + my @res; + foreach my $name (@_) { + next if $seen{$name}; + $seen{$name} = 1; + push @res, $name; + } + return @res; +} + +sub writeFile { + my ($fn, $s) = @_; + open TMP, ">$fn" or die "cannot create file '$fn': $!"; + print TMP "$s" or die; + close TMP or die; +} + +sub readFile { + local $/ = undef; + my ($fn) = @_; + open TMP, "<$fn" or die "cannot open file '$fn': $!"; + my $s = ; + close TMP or die; + return $s; +} + +sub mkTempDir { + my ($name) = @_; + return tempdir("$name.XXXXXX", CLEANUP => 1, DIR => $ENV{"TMPDIR"} // $ENV{"XDG_RUNTIME_DIR"} // "/tmp") + || die "cannot create a temporary directory"; +} diff --git a/third_party/nix/perl/local.mk b/third_party/nix/perl/local.mk new file mode 100644 index 000000000..b13d4c0d6 --- /dev/null +++ b/third_party/nix/perl/local.mk @@ -0,0 +1,43 @@ +nix_perl_sources := \ + lib/Nix/Store.pm \ + lib/Nix/Manifest.pm \ + lib/Nix/SSH.pm \ + lib/Nix/CopyClosure.pm \ + lib/Nix/Config.pm.in \ + lib/Nix/Utils.pm + +nix_perl_modules := $(nix_perl_sources:.in=) + +$(foreach x, $(nix_perl_modules), $(eval $(call install-data-in, $(x), $(perllibdir)/Nix))) + +lib/Nix/Store.cc: lib/Nix/Store.xs + $(trace-gen) xsubpp $^ -output $@ + +libraries += Store + +Store_DIR := lib/Nix + +Store_SOURCES := $(Store_DIR)/Store.cc + +Store_CXXFLAGS = \ + $(NIX_CFLAGS) \ + -I$(shell perl -e 'use Config; print $$Config{archlibexp};')/CORE \ + -D_FILE_OFFSET_BITS=64 \ + -Wno-unknown-warning-option -Wno-unused-variable -Wno-literal-suffix \ + -Wno-reserved-user-defined-literal -Wno-duplicate-decl-specifier -Wno-pointer-bool-conversion + +Store_LDFLAGS := $(SODIUM_LIBS) $(NIX_LIBS) + +ifeq (CYGWIN,$(findstring CYGWIN,$(OS))) + archlib = $(shell perl -E 'use Config; print $$Config{archlib};') + libperl = $(shell perl -E 'use Config; print $$Config{libperl};') + Store_LDFLAGS += $(shell find ${archlib} -name ${libperl}) +endif + +Store_ALLOW_UNDEFINED = 1 + +Store_FORCE_INSTALL = 1 + +Store_INSTALL_DIR = $(perllibdir)/auto/Nix/Store + +clean-files += lib/Nix/Config.pm lib/Nix/Store.cc Makefile.config diff --git a/third_party/nix/release-common.nix b/third_party/nix/release-common.nix new file mode 100644 index 000000000..d66bbafa8 --- /dev/null +++ b/third_party/nix/release-common.nix @@ -0,0 +1,83 @@ +{ pkgs }: + +with pkgs; + +rec { + # Use "busybox-sandbox-shell" if present, + # if not (legacy) fallback and hope it's sufficient. + sh = pkgs.busybox-sandbox-shell or (busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + configureFlags = + [ + "--enable-gc" + ] ++ lib.optionals stdenv.isLinux [ + "--with-sandbox-shell=${sh}/bin/busybox" + ]; + + tarballDeps = + [ bison + flex + libxml2 + libxslt + docbook5 + docbook_xsl_ns + autoconf-archive + autoreconfHook + ]; + + buildDeps = + [ curl + bzip2 xz brotli editline + openssl pkgconfig sqlite + boost + + # Tests + git + mercurial + ] + ++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) + ((aws-sdk-cpp.override { + apis = ["s3" "transfer"]; + customMemoryManagement = false; + }).overrideDerivation (args: { + /* + patches = args.patches or [] ++ [ (fetchpatch { + url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch; + sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2"; + }) ]; + */ + })); + + propagatedDeps = + [ (boehmgc.override { enableLargeConfig = true; }) + ]; + + perlDeps = + [ perl + perlPackages.DBDSQLite + ]; +} diff --git a/third_party/nix/release.nix b/third_party/nix/release.nix new file mode 100644 index 000000000..64aa1a976 --- /dev/null +++ b/third_party/nix/release.nix @@ -0,0 +1,397 @@ +{ nix ? builtins.fetchGit ./. +, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz +, officialRelease ? false +, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ] +}: + +let + + pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; }; + + jobs = rec { + + + tarball = + with pkgs; + + with import ./release-common.nix { inherit pkgs; }; + + releaseTools.sourceTarball { + name = "nix-tarball"; + version = builtins.readFile ./.version; + versionSuffix = if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}"; + src = nix; + inherit officialRelease; + + buildInputs = tarballDeps ++ buildDeps ++ propagatedDeps; + + configureFlags = "--enable-gc"; + + postUnpack = '' + (cd $sourceRoot && find . -type f) | cut -c3- > $sourceRoot/.dist-files + cat $sourceRoot/.dist-files + ''; + + preConfigure = '' + (cd perl ; autoreconf --install --force --verbose) + # TeX needs a writable font cache. + export VARTEXFONTS=$TMPDIR/texfonts + ''; + + distPhase = + '' + runHook preDist + make dist + mkdir -p $out/tarballs + cp *.tar.* $out/tarballs + ''; + + preDist = '' + make install docdir=$out/share/doc/nix makefiles=doc/manual/local.mk + echo "doc manual $out/share/doc/nix/manual" >> $out/nix-support/hydra-build-products + ''; + }; + + + build = pkgs.lib.genAttrs systems (system: + + let pkgs = import nixpkgs { inherit system; }; in + + with pkgs; + + with import ./release-common.nix { inherit pkgs; }; + + releaseTools.nixBuild { + name = "nix"; + src = tarball; + + buildInputs = buildDeps; + + propagatedBuildInputs = propagatedDeps; + + preConfigure = + # Copy libboost_context so we don't get all of Boost in our closure. + # https://github.com/NixOS/nixpkgs/issues/45462 + '' + mkdir -p $out/lib + cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib + rm -f $out/lib/*.a + ${lib.optionalString stdenv.isLinux '' + chmod u+w $out/lib/*.so.* + patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + ''} + ''; + + configureFlags = configureFlags ++ + [ "--sysconfdir=/etc" ]; + + enableParallelBuilding = true; + + makeFlags = "profiledir=$(out)/etc/profile.d"; + + installFlags = "sysconfdir=$(out)/etc"; + + doInstallCheck = true; + installCheckFlags = "sysconfdir=$(out)/etc"; + + separateDebugInfo = true; + }); + + + perlBindings = pkgs.lib.genAttrs systems (system: + + let pkgs = import nixpkgs { inherit system; }; in with pkgs; + + releaseTools.nixBuild { + name = "nix-perl"; + src = tarball; + + buildInputs = + [ jobs.build.${system} curl bzip2 xz pkgconfig pkgs.perl boost ] + ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium; + + configureFlags = '' + --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} + --with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix} + ''; + + enableParallelBuilding = true; + + postUnpack = "sourceRoot=$sourceRoot/perl"; + }); + + + binaryTarball = pkgs.lib.genAttrs systems (system: + + with import nixpkgs { inherit system; }; + + let + toplevel = builtins.getAttr system jobs.build; + version = toplevel.src.version; + installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; }; + in + + runCommand "nix-binary-tarball-${version}" + { #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck; + meta.description = "Distribution-independent Nix bootstrap binaries for ${system}"; + } + '' + cp ${installerClosureInfo}/registration $TMPDIR/reginfo + substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + + substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \ + --subst-var-by nix ${toplevel} \ + --subst-var-by cacert ${cacert} + + if type -p shellcheck; then + # SC1090: Don't worry about not being able to find + # $nix/etc/profile.d/nix.sh + shellcheck --exclude SC1090 $TMPDIR/install + shellcheck $TMPDIR/install-darwin-multi-user.sh + shellcheck $TMPDIR/install-systemd-multi-user.sh + + # SC1091: Don't panic about not being able to source + # /etc/profile + # SC2002: Ignore "useless cat" "error", when loading + # .reginfo, as the cat is a much cleaner + # implementation, even though it is "useless" + # SC2116: Allow ROOT_HOME=$(echo ~root) for resolving + # root's home directory + shellcheck --external-sources \ + --exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user + fi + + chmod +x $TMPDIR/install + chmod +x $TMPDIR/install-darwin-multi-user.sh + chmod +x $TMPDIR/install-systemd-multi-user.sh + chmod +x $TMPDIR/install-multi-user + dir=nix-${version}-${system} + fn=$out/$dir.tar.xz + mkdir -p $out/nix-support + echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products + tar cvfJ $fn \ + --owner=0 --group=0 --mode=u+rw,uga+r \ + --absolute-names \ + --hard-dereference \ + --transform "s,$TMPDIR/install,$dir/install," \ + --transform "s,$TMPDIR/reginfo,$dir/.reginfo," \ + --transform "s,$NIX_STORE,$dir/store,S" \ + $TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \ + $TMPDIR/install-systemd-multi-user.sh \ + $TMPDIR/install-multi-user $TMPDIR/reginfo \ + $(cat ${installerClosureInfo}/store-paths) + ''); + + + coverage = + with pkgs; + + with import ./release-common.nix { inherit pkgs; }; + + releaseTools.coverageAnalysis { + name = "nix-build"; + src = tarball; + + enableParallelBuilding = true; + + buildInputs = buildDeps ++ propagatedDeps; + + dontInstall = false; + + doInstallCheck = true; + + lcovFilter = [ "*/boost/*" "*-tab.*" "*/nlohmann/*" "*/linenoise/*" ]; + + # We call `dot', and even though we just use it to + # syntax-check generated dot files, it still requires some + # fonts. So provide those. + FONTCONFIG_FILE = texFunctions.fontsConf; + }; + + + #rpm_fedora27x86_64 = makeRPM_x86_64 (diskImageFunsFun: diskImageFunsFun.fedora27x86_64) [ ]; + + + #deb_debian8i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.debian8i386) [ "libsodium-dev" ] [ "libsodium13" ]; + #deb_debian8x86_64 = makeDeb_x86_64 (diskImageFunsFun: diskImageFunsFun.debian8x86_64) [ "libsodium-dev" ] [ "libsodium13" ]; + + #deb_ubuntu1710i386 = makeDeb_i686 (diskImageFuns: diskImageFuns.ubuntu1710i386) [ ] [ "libsodium18" ]; + #deb_ubuntu1710x86_64 = makeDeb_x86_64 (diskImageFuns: diskImageFuns.ubuntu1710x86_64) [ ] [ "libsodium18" "libboost-context1.62.0" ]; + + + # System tests. + tests.remoteBuilds = (import ./tests/remote-builds.nix rec { + inherit nixpkgs; + nix = build.x86_64-linux; system = "x86_64-linux"; + }); + + tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec { + inherit nixpkgs; + nix = build.x86_64-linux; system = "x86_64-linux"; + }); + + tests.setuid = pkgs.lib.genAttrs + ["i686-linux" "x86_64-linux"] + (system: + import ./tests/setuid.nix rec { + inherit nixpkgs; + nix = build.${system}; inherit system; + }); + + tests.binaryTarball = + with import nixpkgs { system = "x86_64-linux"; }; + vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test" + { diskImage = vmTools.diskImages.ubuntu1204x86_64; + } + '' + set -x + useradd -m alice + su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*' + mkdir /dest-nix + mount -o bind /dest-nix /nix # Provide a writable /nix. + chown alice /nix + su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install' + su - alice -c 'nix-store --verify' + su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}' + + # Check whether 'nix upgrade-nix' works. + cat > /tmp/paths.nix <> $out/nix-support/hydra-build-products + ''; + + + # Aggregate job containing the release-critical jobs. + release = pkgs.releaseTools.aggregate { + name = "nix-${tarball.version}"; + meta.description = "Release-critical builds"; + constituents = + [ tarball + build.i686-linux + build.x86_64-darwin + build.x86_64-linux + build.aarch64-linux + binaryTarball.i686-linux + binaryTarball.x86_64-darwin + binaryTarball.x86_64-linux + binaryTarball.aarch64-linux + tests.remoteBuilds + tests.nix-copy-closure + tests.binaryTarball + #tests.evalNixpkgs + #tests.evalNixOS + installerScript + ]; + }; + + }; + + + makeRPM_i686 = makeRPM "i686-linux"; + makeRPM_x86_64 = makeRPM "x86_64-linux"; + + makeRPM = + system: diskImageFun: extraPackages: + + with import nixpkgs { inherit system; }; + + releaseTools.rpmBuild rec { + name = "nix-rpm"; + src = jobs.tarball; + diskImage = (diskImageFun vmTools.diskImageFuns) + { extraPackages = + [ "sqlite" "sqlite-devel" "bzip2-devel" "libcurl-devel" "openssl-devel" "xz-devel" "libseccomp-devel" "libsodium-devel" "boost-devel" "bison" "flex" ] + ++ extraPackages; }; + # At most 2047MB can be simulated in qemu-system-i386 + memSize = 2047; + meta.schedulingPriority = 50; + postRPMInstall = "cd /tmp/rpmout/BUILD/nix-* && make installcheck"; + #enableParallelBuilding = true; + }; + + + makeDeb_i686 = makeDeb "i686-linux"; + makeDeb_x86_64 = makeDeb "x86_64-linux"; + + makeDeb = + system: diskImageFun: extraPackages: extraDebPackages: + + with import nixpkgs { inherit system; }; + + releaseTools.debBuild { + name = "nix-deb"; + src = jobs.tarball; + diskImage = (diskImageFun vmTools.diskImageFuns) + { extraPackages = + [ "libsqlite3-dev" "libbz2-dev" "libcurl-dev" "libcurl3-nss" "libssl-dev" "liblzma-dev" "libseccomp-dev" "libsodium-dev" "libboost-all-dev" ] + ++ extraPackages; }; + memSize = 2047; + meta.schedulingPriority = 50; + postInstall = "make installcheck"; + configureFlags = "--sysconfdir=/etc"; + debRequires = + [ "curl" "libsqlite3-0" "libbz2-1.0" "bzip2" "xz-utils" "libssl1.0.0" "liblzma5" "libseccomp2" ] + ++ extraDebPackages; + debMaintainer = "Eelco Dolstra "; + doInstallCheck = true; + #enableParallelBuilding = true; + }; + + +in jobs diff --git a/third_party/nix/scripts/install-darwin-multi-user.sh b/third_party/nix/scripts/install-darwin-multi-user.sh new file mode 100644 index 000000000..49076bd5c --- /dev/null +++ b/third_party/nix/scripts/install-darwin-multi-user.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +readonly PLIST_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist + +dsclattr() { + /usr/bin/dscl . -read "$1" \ + | awk "/$2/ { print \$2 }" +} + +poly_validate_assumptions() { + if [ "$(uname -s)" != "Darwin" ]; then + failure "This script is for use with macOS!" + fi +} + +poly_service_installed_check() { + [ -e "$PLIST_DEST" ] +} + +poly_service_uninstall_directions() { + cat < /dev/null 2>&1 +} + +poly_group_id_get() { + dsclattr "/Groups/$1" "PrimaryGroupID" +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + /usr/sbin/dseditgroup -o create \ + -r "Nix build group for nix-daemon" \ + -i "$NIX_BUILD_GROUP_ID" \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + /usr/bin/dscl . -read "/Users/$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + dsclattr "/Users/$1" "UniqueID" +} + +poly_user_hidden_get() { + dsclattr "/Users/$1" "IsHidden" +} + +poly_user_hidden_set() { + _sudo "in order to make $1 a hidden user" \ + /usr/bin/dscl . -create "/Users/$1" "IsHidden" "1" +} + +poly_user_home_get() { + dsclattr "/Users/$1" "NFSHomeDirectory" +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "NFSHomeDirectory" "$2" +} + +poly_user_note_get() { + dsclattr "/Users/$1" "RealName" +} + +poly_user_note_set() { + _sudo "in order to give $username a useful note" \ + /usr/bin/dscl . -create "/Users/$1" "RealName" "$2" +} + +poly_user_shell_get() { + dsclattr "/Users/$1" "UserShell" +} + +poly_user_shell_set() { + _sudo "in order to give $1 a safe home directory" \ + /usr/bin/dscl . -create "/Users/$1" "UserShell" "$2" +} + +poly_user_in_group_check() { + username=$1 + group=$2 + dseditgroup -o checkmember -m "$username" "$group" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + username=$1 + group=$2 + + _sudo "Add $username to the $group group"\ + /usr/sbin/dseditgroup -o edit -t user \ + -a "$username" "$group" +} + +poly_user_primary_group_get() { + dsclattr "/Users/$1" "PrimaryGroupID" +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + /usr/bin/dscl . -create "/Users/$1" "PrimaryGroupID" "$2" +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user (#$builder_num), $username" \ + /usr/bin/dscl . create "/Users/$username" \ + UniqueID "${uid}" +} diff --git a/third_party/nix/scripts/install-multi-user.sh b/third_party/nix/scripts/install-multi-user.sh new file mode 100644 index 000000000..5233762fa --- /dev/null +++ b/third_party/nix/scripts/install-multi-user.sh @@ -0,0 +1,798 @@ +#!/usr/bin/env bash + +set -eu +set -o pipefail + +# Sourced from: +# - https://github.com/LnL7/nix-darwin/blob/8c29d0985d74b4a990238497c47a2542a5616b3c/bootstrap.sh +# - https://gist.github.com/expipiplus1/e571ce88c608a1e83547c918591b149f/ac504c6c1b96e65505fbda437a28ce563408ecb0 +# - https://github.com/NixOS/nixos-org-configurations/blob/a122f418797713d519aadf02e677fce0dc1cb446/delft/scripts/nix-mac-installer.sh +# - https://github.com/matthewbauer/macNixOS/blob/f6045394f9153edea417be90c216788e754feaba/install-macNixOS.sh +# - https://gist.github.com/LnL7/9717bd6cdcb30b086fd7f2093e5f8494/86b26f852ce563e973acd30f796a9a416248c34a +# +# however tracking which bits came from which would be impossible. + +readonly ESC='\033[0m' +readonly BOLD='\033[1m' +readonly BLUE='\033[34m' +readonly BLUE_UL='\033[4;34m' +readonly GREEN='\033[32m' +readonly GREEN_UL='\033[4;32m' +readonly RED='\033[31m' + +readonly NIX_USER_COUNT="32" +readonly NIX_BUILD_GROUP_ID="30000" +readonly NIX_BUILD_GROUP_NAME="nixbld" +readonly NIX_FIRST_BUILD_UID="30001" +# Please don't change this. We don't support it, because the +# default shell profile that comes with Nix doesn't support it. +readonly NIX_ROOT="/nix" + +readonly PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc") +readonly PROFILE_BACKUP_SUFFIX=".backup-before-nix" +readonly PROFILE_NIX_FILE="$NIX_ROOT/var/nix/profiles/default/etc/profile.d/nix-daemon.sh" + +readonly NIX_INSTALLED_NIX="@nix@" +readonly NIX_INSTALLED_CACERT="@cacert@" +readonly EXTRACTED_NIX_PATH="$(dirname "$0")" + +readonly ROOT_HOME=$(echo ~root) + +if [ -t 0 ]; then + readonly IS_HEADLESS='no' +else + readonly IS_HEADLESS='yes' +fi + +headless() { + if [ "$IS_HEADLESS" = "yes" ]; then + return 0 + else + return 1 + fi +} + +contactme() { + echo "We'd love to help if you need it." + echo "" + echo "If you can, open an issue at https://github.com/nixos/nix/issues" + echo "" + echo "Or feel free to contact the team," + echo " - on IRC #nixos on irc.freenode.net" + echo " - on twitter @nixos_org" +} + +uninstall_directions() { + subheader "Uninstalling nix:" + local step=0 + + if poly_service_installed_check; then + step=$((step + 1)) + poly_service_uninstall_directions "$step" + fi + + for profile_target in "${PROFILE_TARGETS[@]}"; do + if [ -e "$profile_target" ] && [ -e "$profile_target$PROFILE_BACKUP_SUFFIX" ]; then + step=$((step + 1)) + cat < $1" +} + +bold() { + echo "$BOLD$*$ESC" +} + +ok() { + _textout "$GREEN" "$@" +} + +warning() { + warningheader "warning!" + cat + echo "" +} + +failure() { + header "oh no!" + _textout "$RED" "$@" + echo "" + _textout "$RED" "$(contactme)" + trap finish_cleanup EXIT + exit 1 +} + +ui_confirm() { + _textout "$GREEN$GREEN_UL" "$1" + + if headless; then + echo "No TTY, assuming you would say yes :)" + return 0 + fi + + local prompt="[y/n] " + echo -n "$prompt" + while read -r y; do + if [ "$y" = "y" ]; then + echo "" + return 0 + elif [ "$y" = "n" ]; then + echo "" + return 1 + else + _textout "$RED" "Sorry, I didn't understand. I can only understand answers of y or n" + echo -n "$prompt" + fi + done + echo "" + return 1 +} + +__sudo() { + local expl="$1" + local cmd="$2" + shift + header "sudo execution" + + echo "I am executing:" + echo "" + printf " $ sudo %s\\n" "$cmd" + echo "" + echo "$expl" + echo "" + + return 0 +} + +_sudo() { + local expl="$1" + shift + if ! headless; then + __sudo "$expl" "$*" + fi + sudo "$@" +} + + +readonly SCRATCH=$(mktemp -d -t tmp.XXXXXXXXXX) +function finish_cleanup { + rm -rf "$SCRATCH" +} + +function finish_fail { + finish_cleanup + + failure < /dev/null >&2; then + failure < "$SCRATCH/.nix-channels" + _sudo "to set up the default system channel (part 1)" \ + install -m 0664 "$SCRATCH/.nix-channels" "$ROOT_HOME/.nix-channels" +} + +welcome_to_nix() { + ok "Welcome to the Multi-User Nix Installation" + + cat < "$SCRATCH/nix.conf" +build-users-group = $NIX_BUILD_GROUP_NAME +EOF + _sudo "to place the default nix daemon configuration (part 2)" \ + install -m 0664 "$SCRATCH/nix.conf" /etc/nix/nix.conf +} + +main() { + if [ "$(uname -s)" = "Darwin" ]; then + # shellcheck source=./install-darwin-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-darwin-multi-user.sh" + elif [ "$(uname -s)" = "Linux" ]; then + if [ -e /run/systemd/system ]; then + # shellcheck source=./install-systemd-multi-user.sh + . "$EXTRACTED_NIX_PATH/install-systemd-multi-user.sh" + else + failure "Sorry, the multi-user installation requires systemd on Linux (detected using /run/systemd/system)" + fi + else + failure "Sorry, I don't know what to do on $(uname)" + fi + + welcome_to_nix + chat_about_sudo + + if [ "${ALLOW_PREEXISTING_INSTALLATION:-}" = "" ]; then + validate_starting_assumptions + fi + + setup_report + + if ! ui_confirm "Ready to continue?"; then + ok "Alright, no changes have been made :)" + contactme + trap finish_cleanup EXIT + exit 1 + fi + + create_build_group + create_build_users + create_directories + place_channel_configuration + install_from_extracted_nix + + configure_shell_profile + + set +eu + . /etc/profile + set -eu + + setup_default_profile + place_nix_configuration + poly_configure_nix_daemon_service + + trap finish_success EXIT +} + + +main diff --git a/third_party/nix/scripts/install-nix-from-closure.sh b/third_party/nix/scripts/install-nix-from-closure.sh new file mode 100644 index 000000000..3f1581854 --- /dev/null +++ b/third_party/nix/scripts/install-nix-from-closure.sh @@ -0,0 +1,180 @@ +#!/bin/sh + +set -e + +dest="/nix" +self="$(dirname "$0")" +nix="@nix@" +cacert="@cacert@" + + +if ! [ -e "$self/.reginfo" ]; then + echo "$0: incomplete installer (.reginfo is missing)" >&2 +fi + +if [ -z "$USER" ] && ! USER=$(id -u -n); then + echo "$0: \$USER is not set" >&2 + exit 1 +fi + +if [ -z "$HOME" ]; then + echo "$0: \$HOME is not set" >&2 + exit 1 +fi + +# macOS support for 10.12.6 or higher +if [ "$(uname -s)" = "Darwin" ]; then + macos_major=$(sw_vers -productVersion | cut -d '.' -f 2) + macos_minor=$(sw_vers -productVersion | cut -d '.' -f 3) + if [ "$macos_major" -lt 12 ] || { [ "$macos_major" -eq 12 ] && [ "$macos_minor" -lt 6 ]; }; then + echo "$0: macOS $(sw_vers -productVersion) is not supported, upgrade to 10.12.6 or higher" + exit 1 + fi +fi + +# Determine if we could use the multi-user installer or not +if [ "$(uname -s)" = "Darwin" ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +elif [ "$(uname -s)" = "Linux" ] && [ -e /run/systemd/system ]; then + echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 +fi + +INSTALL_MODE=no-daemon +# Trivially handle the --daemon / --no-daemon options +if [ "x${1:-}" = "x--no-daemon" ]; then + INSTALL_MODE=no-daemon +elif [ "x${1:-}" = "x--daemon" ]; then + INSTALL_MODE=daemon +elif [ "x${1:-}" != "x" ]; then + ( + echo "Nix Installer [--daemon|--no-daemon]" + + echo "Choose installation method." + echo "" + echo " --daemon: Installs and configures a background daemon that manages the store," + echo " providing multi-user support and better isolation for local builds." + echo " Both for security and reproducibility, this method is recommended if" + echo " supported on your platform." + echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo "" + echo " --no-daemon: Simple, single-user installation that does not require root and is" + echo " trivial to uninstall." + echo " (default)" + echo "" + ) >&2 + exit +fi + +if [ "$INSTALL_MODE" = "daemon" ]; then + printf '\e[1;31mSwitching to the Daemon-based Installer\e[0m\n' + exec "$self/install-multi-user" + exit 0 +fi + +if [ "$(id -u)" -eq 0 ]; then + printf '\e[1;31mwarning: installing Nix as root is not supported by this script!\e[0m\n' +fi + +echo "performing a single-user installation of Nix..." >&2 + +if ! [ -e $dest ]; then + cmd="mkdir -m 0755 $dest && chown $USER $dest" + echo "directory $dest does not exist; creating it by running '$cmd' using sudo" >&2 + if ! sudo sh -c "$cmd"; then + echo "$0: please manually run '$cmd' as root to create $dest" >&2 + exit 1 + fi +fi + +if ! [ -w $dest ]; then + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see http://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 + exit 1 +fi + +mkdir -p $dest/store + +printf "copying Nix to %s..." "${dest}/store" >&2 + +for i in $(cd "$self/store" >/dev/null && echo ./*); do + printf "." >&2 + i_tmp="$dest/store/$i.$$" + if [ -e "$i_tmp" ]; then + rm -rf "$i_tmp" + fi + if ! [ -e "$dest/store/$i" ]; then + cp -Rp "$self/store/$i" "$i_tmp" + chmod -R a-w "$i_tmp" + chmod +w "$i_tmp" + mv "$i_tmp" "$dest/store/$i" + chmod -w "$dest/store/$i" + fi +done +echo "" >&2 + +if ! "$nix/bin/nix-store" --load-db < "$self/.reginfo"; then + echo "$0: unable to register valid paths" >&2 + exit 1 +fi + +. "$nix/etc/profile.d/nix.sh" + +if ! "$nix/bin/nix-env" -i "$nix"; then + echo "$0: unable to install Nix into your default profile" >&2 + exit 1 +fi + +# Install an SSL certificate bundle. +if [ -z "$NIX_SSL_CERT_FILE" ] || ! [ -f "$NIX_SSL_CERT_FILE" ]; then + $nix/bin/nix-env -i "$cacert" + export NIX_SSL_CERT_FILE="$HOME/.nix-profile/etc/ssl/certs/ca-bundle.crt" +fi + +# Subscribe the user to the Nixpkgs channel and fetch it. +if ! $nix/bin/nix-channel --list | grep -q "^nixpkgs "; then + $nix/bin/nix-channel --add https://nixos.org/channels/nixpkgs-unstable +fi +if [ -z "$_NIX_INSTALLER_TEST" ]; then + if ! $nix/bin/nix-channel --update nixpkgs; then + echo "Fetching the nixpkgs channel failed. (Are you offline?)" + echo "To try again later, run \"nix-channel --update nixpkgs\"." + fi +fi + +added= +p=$HOME/.nix-profile/etc/profile.d/nix.sh +if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then + # Make the shell source nix.sh during login. + for i in .bash_profile .bash_login .profile; do + fn="$HOME/$i" + if [ -w "$fn" ]; then + if ! grep -q "$p" "$fn"; then + echo "modifying $fn..." >&2 + echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn" + fi + added=1 + break + fi + done +fi + +if [ -z "$added" ]; then + cat >&2 <&2 < /dev/null 2>&1 +} + +poly_group_id_get() { + getent group "$1" | cut -d: -f3 +} + +poly_create_build_group() { + _sudo "Create the Nix build group, $NIX_BUILD_GROUP_NAME" \ + groupadd -g "$NIX_BUILD_GROUP_ID" --system \ + "$NIX_BUILD_GROUP_NAME" >&2 +} + +poly_user_exists() { + getent passwd "$1" > /dev/null 2>&1 +} + +poly_user_id_get() { + getent passwd "$1" | cut -d: -f3 +} + +poly_user_hidden_get() { + echo "1" +} + +poly_user_hidden_set() { + true +} + +poly_user_home_get() { + getent passwd "$1" | cut -d: -f6 +} + +poly_user_home_set() { + _sudo "in order to give $1 a safe home directory" \ + usermod --home "$2" "$1" +} + +poly_user_note_get() { + getent passwd "$1" | cut -d: -f5 +} + +poly_user_note_set() { + _sudo "in order to give $1 a useful comment" \ + usermod --comment "$2" "$1" +} + +poly_user_shell_get() { + getent passwd "$1" | cut -d: -f7 +} + +poly_user_shell_set() { + _sudo "in order to prevent $1 from logging in" \ + usermod --shell "$2" "$1" +} + +poly_user_in_group_check() { + groups "$1" | grep -q "$2" > /dev/null 2>&1 +} + +poly_user_in_group_set() { + _sudo "Add $1 to the $2 group"\ + usermod --append --groups "$2" "$1" +} + +poly_user_primary_group_get() { + getent passwd "$1" | cut -d: -f4 +} + +poly_user_primary_group_set() { + _sudo "to let the nix daemon use this user for builds (this might seem redundant, but there are two concepts of group membership)" \ + usermod --gid "$2" "$1" + +} + +poly_create_build_user() { + username=$1 + uid=$2 + builder_num=$3 + + _sudo "Creating the Nix build user, $username" \ + useradd \ + --home-dir /var/empty \ + --comment "Nix build user $builder_num" \ + --gid "$NIX_BUILD_GROUP_ID" \ + --groups "$NIX_BUILD_GROUP_NAME" \ + --no-user-group \ + --system \ + --shell /sbin/nologin \ + --uid "$uid" \ + --password "!" \ + "$username" +} diff --git a/third_party/nix/scripts/install.in b/third_party/nix/scripts/install.in new file mode 100644 index 000000000..902758b13 --- /dev/null +++ b/third_party/nix/scripts/install.in @@ -0,0 +1,66 @@ +#!/bin/sh + +# This script installs the Nix package manager on your system by +# downloading a binary distribution and running its installer script +# (which in turn creates and populates /nix). + +{ # Prevent execution if this script was only partially downloaded +oops() { + echo "$0:" "$@" >&2 + exit 1 +} + +tmpDir="$(mktemp -d -t nix-binary-tarball-unpack.XXXXXXXXXX || \ + oops "Can't create temporary directory for downloading the Nix binary tarball")" +cleanup() { + rm -rf "$tmpDir" +} +trap cleanup EXIT INT QUIT TERM + +require_util() { + command -v "$1" > /dev/null 2>&1 || + oops "you do not have '$1' installed, which I need to $2" +} + +case "$(uname -s).$(uname -m)" in + Linux.x86_64) system=x86_64-linux; hash=@binaryTarball_x86_64-linux@;; + Linux.i?86) system=i686-linux; hash=@binaryTarball_i686-linux@;; + Linux.aarch64) system=aarch64-linux; hash=@binaryTarball_aarch64-linux@;; + Darwin.x86_64) system=x86_64-darwin; hash=@binaryTarball_x86_64-darwin@;; + *) oops "sorry, there is no binary distribution of Nix for your platform";; +esac + +url="https://nixos.org/releases/nix/nix-@nixVersion@/nix-@nixVersion@-$system.tar.xz" + +tarball="$tmpDir/$(basename "$tmpDir/nix-@nixVersion@-$system.tar.xz")" + +require_util curl "download the binary tarball" +require_util tar "unpack the binary tarball" + +echo "downloading Nix @nixVersion@ binary tarball for $system from '$url' to '$tmpDir'..." +curl -L "$url" -o "$tarball" || oops "failed to download '$url'" + +if command -v sha256sum > /dev/null 2>&1; then + hash2="$(sha256sum -b "$tarball" | cut -c1-64)" +elif command -v shasum > /dev/null 2>&1; then + hash2="$(shasum -a 256 -b "$tarball" | cut -c1-64)" +elif command -v openssl > /dev/null 2>&1; then + hash2="$(openssl dgst -r -sha256 "$tarball" | cut -c1-64)" +else + oops "cannot verify the SHA-256 hash of '$url'; you need one of 'shasum', 'sha256sum', or 'openssl'" +fi + +if [ "$hash" != "$hash2" ]; then + oops "SHA-256 hash mismatch in '$url'; expected $hash, got $hash2" +fi + +unpack=$tmpDir/unpack +mkdir -p "$unpack" +tar -xf "$tarball" -C "$unpack" || oops "failed to unpack '$url'" + +script=$(echo "$unpack"/*/install) + +[ -e "$script" ] || oops "installation script is missing from the binary tarball!" +"$script" "$@" + +} # End of wrapping diff --git a/third_party/nix/scripts/local.mk b/third_party/nix/scripts/local.mk new file mode 100644 index 000000000..2a0055852 --- /dev/null +++ b/third_party/nix/scripts/local.mk @@ -0,0 +1,13 @@ +nix_noinst_scripts := \ + $(d)/nix-http-export.cgi \ + $(d)/nix-profile.sh \ + $(d)/nix-reduce-build + +noinst-scripts += $(nix_noinst_scripts) + +profiledir = $(sysconfdir)/profile.d + +$(eval $(call install-file-as, $(d)/nix-profile.sh, $(profiledir)/nix.sh, 0644)) +$(eval $(call install-file-as, $(d)/nix-profile-daemon.sh, $(profiledir)/nix-daemon.sh, 0644)) + +clean-files += $(nix_noinst_scripts) diff --git a/third_party/nix/scripts/nix-http-export.cgi.in b/third_party/nix/scripts/nix-http-export.cgi.in new file mode 100755 index 000000000..19a505af1 --- /dev/null +++ b/third_party/nix/scripts/nix-http-export.cgi.in @@ -0,0 +1,51 @@ +#! /bin/sh + +export HOME=/tmp +export NIX_REMOTE=daemon + +TMP_DIR="${TMP_DIR:-/tmp/nix-export}" + +@coreutils@/mkdir -p "$TMP_DIR" || true +@coreutils@/chmod a+r "$TMP_DIR" + +needed_path="?$QUERY_STRING" +needed_path="${needed_path#*[?&]needed_path=}" +needed_path="${needed_path%%&*}" +#needed_path="$(echo $needed_path | ./unhttp)" +needed_path="${needed_path//%2B/+}" +needed_path="${needed_path//%3D/=}" + +echo needed_path: "$needed_path" >&2 + +NIX_STORE="${NIX_STORE_DIR:-/nix/store}" + +echo NIX_STORE: "${NIX_STORE}" >&2 + +full_path="${NIX_STORE}"/"$needed_path" + +if [ "$needed_path" != "${needed_path%.drv}" ]; then + echo "Status: 403 You should create the derivation file yourself" + echo "Content-Type: text/plain" + echo + echo "Refusing to disclose derivation contents" + exit +fi + +if @bindir@/nix-store --check-validity "$full_path"; then + if ! [ -e nix-export/"$needed_path".nar.gz ]; then + @bindir@/nix-store --export "$full_path" | @gzip@ > "$TMP_DIR"/"$needed_path".nar.gz + @coreutils@/ln -fs "$TMP_DIR"/"$needed_path".nar.gz nix-export/"$needed_path".nar.gz + fi; + echo "Status: 301 Moved" + echo "Location: nix-export/"$needed_path".nar.gz" + echo +else + echo "Status: 404 No such path found" + echo "Content-Type: text/plain" + echo + echo "Path not found:" + echo "$needed_path" + echo "checked:" + echo "$full_path" +fi + diff --git a/third_party/nix/scripts/nix-profile-daemon.sh.in b/third_party/nix/scripts/nix-profile-daemon.sh.in new file mode 100644 index 000000000..47655080a --- /dev/null +++ b/third_party/nix/scripts/nix-profile-daemon.sh.in @@ -0,0 +1,29 @@ +# Only execute this file once per shell. +if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi +__ETC_PROFILE_NIX_SOURCED=1 + +export NIX_USER_PROFILE_DIR="@localstatedir@/nix/profiles/per-user/$USER" +export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + +# Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. +if [ ! -z "${NIX_SSL_CERT_FILE:-}" ]; then + : # Allow users to override the NIX_SSL_CERT_FILE +elif [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt +elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem +elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt +elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt +else + # Fall back to what is in the nix profiles, favouring whatever is defined last. + for i in $NIX_PROFILES; do + if [ -e $i/etc/ssl/certs/ca-bundle.crt ]; then + export NIX_SSL_CERT_FILE=$i/etc/ssl/certs/ca-bundle.crt + fi + done +fi + +export NIX_PATH="nixpkgs=@localstatedir@/nix/profiles/per-user/root/channels/nixpkgs:@localstatedir@/nix/profiles/per-user/root/channels" +export PATH="$HOME/.nix-profile/bin:@localstatedir@/nix/profiles/default/bin:$PATH" diff --git a/third_party/nix/scripts/nix-profile.sh.in b/third_party/nix/scripts/nix-profile.sh.in new file mode 100644 index 000000000..e15f7cd46 --- /dev/null +++ b/third_party/nix/scripts/nix-profile.sh.in @@ -0,0 +1,39 @@ +if [ -n "$HOME" ] && [ -n "$USER" ]; then + + # Set up the per-user profile. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/shell.nix + + NIX_LINK=$HOME/.nix-profile + + NIX_USER_PROFILE_DIR=@localstatedir@/nix/profiles/per-user/$USER + + # Append ~/.nix-defexpr/channels to $NIX_PATH so that + # paths work when the user has fetched the Nixpkgs channel. + export NIX_PATH=${NIX_PATH:+$NIX_PATH:}$HOME/.nix-defexpr/channels + + # Set up environment. + # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix + export NIX_PROFILES="@localstatedir@/nix/profiles/default $HOME/.nix-profile" + + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. + if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + elif [ -e /etc/ssl/ca-bundle.pem ]; then # openSUSE Tumbleweed + export NIX_SSL_CERT_FILE=/etc/ssl/ca-bundle.pem + elif [ -e /etc/ssl/certs/ca-bundle.crt ]; then # Old NixOS + export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt + elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS + export NIX_SSL_CERT_FILE=/etc/pki/tls/certs/ca-bundle.crt + elif [ -e "$NIX_LINK/etc/ssl/certs/ca-bundle.crt" ]; then # fall back to cacert in Nix profile + export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ssl/certs/ca-bundle.crt" + elif [ -e "$NIX_LINK/etc/ca-bundle.crt" ]; then # old cacert in Nix profile + export NIX_SSL_CERT_FILE="$NIX_LINK/etc/ca-bundle.crt" + fi + + if [ -n "${MANPATH-}" ]; then + export MANPATH="$NIX_LINK/share/man:$MANPATH" + fi + + export PATH="$NIX_LINK/bin:$PATH" + unset NIX_LINK NIX_USER_PROFILE_DIR +fi diff --git a/third_party/nix/scripts/nix-reduce-build.in b/third_party/nix/scripts/nix-reduce-build.in new file mode 100755 index 000000000..50beb9d10 --- /dev/null +++ b/third_party/nix/scripts/nix-reduce-build.in @@ -0,0 +1,171 @@ +#! @bash@ + +WORKING_DIRECTORY=$(mktemp -d "${TMPDIR:-/tmp}"/nix-reduce-build-XXXXXX); +cd "$WORKING_DIRECTORY"; + +if test -z "$1" || test "a--help" = "a$1" ; then + echo 'nix-reduce-build (paths or Nix expressions) -- (package sources)' >&2 + echo As in: >&2 + echo nix-reduce-build /etc/nixos/nixos -- ssh://user@somewhere.nowhere.example.org >&2 + echo nix-reduce-build /etc/nixos/nixos -- \\ + echo " " \''http://somewhere.nowhere.example.org/nix/nix-http-export.cgi?needed_path='\' >&2 + echo " store path name will be added into the end of the URL" >&2 + echo nix-reduce-build /etc/nixos/nixos -- file://home/user/nar/ >&2 + echo " that should be a directory where gzipped 'nix-store --export' ">&2 + echo " files are located (they should have .nar.gz extension)" >&2 + echo " Or all together: " >&2 + echo -e nix-reduce-build /expr.nix /e2.nix -- \\\\\\\n\ + " ssh://a@b.example.com http://n.example.com/get-nar?q= file://nar/" >&2 + echo " Also supports best-effort local builds of failing expression set:" >&2 + echo "nix-reduce-build /e.nix -- nix-daemon:// nix-self://" >&2 + echo " nix-daemon:// builds using daemon" + echo " nix-self:// builds directly using nix-store from current installation" >&2 + echo " nix-daemon-fixed:// and nix-self-fixed:// do the same, but only for" >&2; + echo "derivations with specified output hash (sha256, sha1 or md5)." >&2 + echo " nix-daemon-substitute:// and nix-self-substitute:// try to substitute" >&2; + echo "maximum amount of paths" >&2; + echo " nix-daemon-build:// and nix-self-build:// try to build (not substitute)" >&2; + echo "maximum amount of paths" >&2; + echo " If no package sources are specified, required paths are listed." >&2; + exit; +fi; + +while ! test "$1" = "--" || test "$1" = "" ; do + echo "$1" >> initial; >&2 + shift; +done +shift; +echo Will work on $(cat initial | wc -l) targets. >&2 + +while read ; do + case "$REPLY" in + ${NIX_STORE_DIR:-/nix/store}/*) + echo "$REPLY" >> paths; >&2 + ;; + *) + ( + IFS=: ; + nix-instantiate $REPLY >> paths; + ); + ;; + esac; +done < initial; +echo Proceeding $(cat paths | wc -l) paths. >&2 + +while read; do + case "$REPLY" in + *.drv) + echo "$REPLY" >> derivers; >&2 + ;; + *) + nix-store --query --deriver "$REPLY" >>derivers; + ;; + esac; +done < paths; +echo Found $(cat derivers | wc -l) derivers. >&2 + +cat derivers | xargs nix-store --query -R > derivers-closure; +echo Proceeding at most $(cat derivers-closure | wc -l) derivers. >&2 + +cat derivers-closure | egrep '[.]drv$' | xargs nix-store --query --outputs > wanted-paths; +cat derivers-closure | egrep -v '[.]drv$' >> wanted-paths; +echo Prepared $(cat wanted-paths | wc -l) paths to get. >&2 + +cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; +echo We need $(cat needed-paths | wc -l) paths. >&2 + +egrep '[.]drv$' derivers-closure > critical-derivers; + +if test -z "$1" ; then + cat needed-paths; +fi; + +refresh_critical_derivers() { + echo "Finding needed derivers..." >&2; + cat critical-derivers | while read; do + if ! (nix-store --query --outputs "$REPLY" | xargs nix-store --check-validity &> /dev/null;); then + echo "$REPLY"; + fi; + done > new-critical-derivers; + mv new-critical-derivers critical-derivers; + echo The needed paths are realized by $(cat critical-derivers | wc -l) derivers. >&2 +} + +build_here() { + cat critical-derivers | while read; do + echo "Realising $REPLY using nix-daemon" >&2 + @bindir@/nix-store -r "${REPLY}" + done; +} + +try_to_substitute(){ + cat needed-paths | while read ; do + echo "Building $REPLY using nix-daemon" >&2 + @bindir@/nix-store -r "${NIX_STORE_DIR:-/nix/store}/${REPLY##*/}" + done; +} + +for i in "$@"; do + sshHost="${i#ssh://}"; + httpHost="${i#http://}"; + httpsHost="${i#https://}"; + filePath="${i#file:/}"; + if [ "$i" != "$sshHost" ]; then + cat needed-paths | while read; do + echo "Getting $REPLY and its closure over ssh" >&2 + nix-copy-closure --from "$sshHost" --gzip "$REPLY" &2 + curl ${BAD_CERTIFICATE:+-k} -L "$i${REPLY##*/}" | gunzip | nix-store --import; + done; + elif [ "$i" != "$filePath" ] ; then + cat needed-paths | while read; do + echo "Installing $REPLY from file" >&2 + gunzip < "$filePath/${REPLY##*/}".nar.gz | nix-store --import; + done; + elif [ "$i" = "nix-daemon://" ] ; then + NIX_REMOTE=daemon try_to_substitute; + refresh_critical_derivers; + NIX_REMOTE=daemon build_here; + elif [ "$i" = "nix-self://" ] ; then + NIX_REMOTE= try_to_substitute; + refresh_critical_derivers; + NIX_REMOTE= build_here; + elif [ "$i" = "nix-daemon-fixed://" ] ; then + refresh_critical_derivers; + + cat critical-derivers | while read; do + if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then + echo "Realising $REPLY using nix-daemon" >&2 + NIX_REMOTE=daemon @bindir@/nix-store -r "${REPLY}" + fi; + done; + elif [ "$i" = "nix-self-fixed://" ] ; then + refresh_critical_derivers; + + cat critical-derivers | while read; do + if egrep '"(md5|sha1|sha256)"' "$REPLY" &>/dev/null; then + echo "Realising $REPLY using direct Nix build" >&2 + NIX_REMOTE= @bindir@/nix-store -r "${REPLY}" + fi; + done; + elif [ "$i" = "nix-daemon-substitute://" ] ; then + NIX_REMOTE=daemon try_to_substitute; + elif [ "$i" = "nix-self-substitute://" ] ; then + NIX_REMOTE= try_to_substitute; + elif [ "$i" = "nix-daemon-build://" ] ; then + refresh_critical_derivers; + NIX_REMOTE=daemon build_here; + elif [ "$i" = "nix-self-build://" ] ; then + refresh_critical_derivers; + NIX_REMOTE= build_here; + fi; + mv needed-paths wanted-paths; + cat wanted-paths | xargs nix-store --check-validity --print-invalid > needed-paths; + echo We still need $(cat needed-paths | wc -l) paths. >&2 +done; + +cd / +rm -r "$WORKING_DIRECTORY" diff --git a/third_party/nix/shell.nix b/third_party/nix/shell.nix new file mode 100644 index 000000000..e8026f8e4 --- /dev/null +++ b/third_party/nix/shell.nix @@ -0,0 +1,25 @@ +{ useClang ? false }: + +with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs-channels/archive/nixos-19.03.tar.gz) {}; + +with import ./release-common.nix { inherit pkgs; }; + +(if useClang then clangStdenv else stdenv).mkDerivation { + name = "nix"; + + buildInputs = buildDeps ++ propagatedDeps ++ tarballDeps ++ perlDeps; + + inherit configureFlags; + + enableParallelBuilding = true; + + installFlags = "sysconfdir=$(out)/etc"; + + shellHook = + '' + export prefix=$(pwd)/inst + configureFlags+=" --prefix=$prefix" + PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH + PATH=$prefix/bin:$PATH + ''; +} diff --git a/third_party/nix/src/build-remote/build-remote.cc b/third_party/nix/src/build-remote/build-remote.cc new file mode 100644 index 000000000..279ae62f6 --- /dev/null +++ b/third_party/nix/src/build-remote/build-remote.cc @@ -0,0 +1,266 @@ +#include +#include +#include +#include +#include +#include +#include +#if __APPLE__ +#include +#endif + +#include "machines.hh" +#include "shared.hh" +#include "pathlocks.hh" +#include "globals.hh" +#include "serialise.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "local-store.hh" +#include "legacy.hh" + +using namespace nix; +using std::cin; + +static void handleAlarm(int sig) { +} + +std::string escapeUri(std::string uri) +{ + std::replace(uri.begin(), uri.end(), '/', '_'); + return uri; +} + +static string currentLoad; + +static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot) +{ + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); +} + +static bool allSupportedLocally(const std::set& requiredFeatures) { + for (auto & feature : requiredFeatures) + if (!settings.systemFeatures.get().count(feature)) return false; + return true; +} + +static int _main(int argc, char * * argv) +{ + { + logger = makeJSONLogger(*logger); + + /* Ensure we don't get any SSH passphrase or host key popups. */ + unsetenv("DISPLAY"); + unsetenv("SSH_ASKPASS"); + + if (argc != 2) + throw UsageError("called without required arguments"); + + verbosity = (Verbosity) std::stoll(argv[1]); + + FdSource source(STDIN_FILENO); + + /* Read the parent's settings. */ + while (readInt(source)) { + auto name = readString(source); + auto value = readString(source); + settings.set(name, value); + } + + settings.maxBuildJobs.set("1"); // hack to make tests with local?root= work + + initPlugins(); + + auto store = openStore().cast(); + + /* It would be more appropriate to use $XDG_RUNTIME_DIR, since + that gets cleared on reboot, but it wouldn't work on macOS. */ + currentLoad = store->stateDir + "/current-load"; + + std::shared_ptr sshStore; + AutoCloseFD bestSlotLock; + + auto machines = getMachines(); + debug("got %d remote builders", machines.size()); + + if (machines.empty()) { + std::cerr << "# decline-permanently\n"; + return 0; + } + + string drvPath; + string storeUri; + + while (true) { + + try { + auto s = readString(source); + if (s != "try") return 0; + } catch (EndOfFile &) { return 0; } + + auto amWilling = readInt(source); + auto neededSystem = readString(source); + source >> drvPath; + auto requiredFeatures = readStrings>(source); + + auto canBuildLocally = amWilling + && ( neededSystem == settings.thisSystem + || settings.extraPlatforms.get().count(neededSystem) > 0) + && allSupportedLocally(requiredFeatures); + + /* Error ignored here, will be caught later */ + mkdir(currentLoad.c_str(), 0777); + + while (true) { + bestSlotLock = -1; + AutoCloseFD lock = openLockFile(currentLoad + "/main-lock", true); + lockFile(lock.get(), ltWrite, true); + + bool rightType = false; + + Machine * bestMachine = nullptr; + unsigned long long bestLoad = 0; + for (auto & m : machines) { + debug("considering building on remote machine '%s'", m.storeUri); + + if (m.enabled && std::find(m.systemTypes.begin(), + m.systemTypes.end(), + neededSystem) != m.systemTypes.end() && + m.allSupported(requiredFeatures) && + m.mandatoryMet(requiredFeatures)) { + rightType = true; + AutoCloseFD free; + unsigned long long load = 0; + for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) { + auto slotLock = openSlotLock(m, slot); + if (lockFile(slotLock.get(), ltWrite, false)) { + if (!free) { + free = std::move(slotLock); + } + } else { + ++load; + } + } + if (!free) { + continue; + } + bool best = false; + if (!bestSlotLock) { + best = true; + } else if (load / m.speedFactor < bestLoad / bestMachine->speedFactor) { + best = true; + } else if (load / m.speedFactor == bestLoad / bestMachine->speedFactor) { + if (m.speedFactor > bestMachine->speedFactor) { + best = true; + } else if (m.speedFactor == bestMachine->speedFactor) { + if (load < bestLoad) { + best = true; + } + } + } + if (best) { + bestLoad = load; + bestSlotLock = std::move(free); + bestMachine = &m; + } + } + } + + if (!bestSlotLock) { + if (rightType && !canBuildLocally) + std::cerr << "# postpone\n"; + else + std::cerr << "# decline\n"; + break; + } + +#if __APPLE__ + futimes(bestSlotLock.get(), NULL); +#else + futimens(bestSlotLock.get(), NULL); +#endif + + lock = -1; + + try { + + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + + Store::Params storeParams; + if (hasPrefix(bestMachine->storeUri, "ssh://")) { + storeParams["max-connections"] ="1"; + storeParams["log-fd"] = "4"; + if (bestMachine->sshKey != "") + storeParams["ssh-key"] = bestMachine->sshKey; + } + + sshStore = openStore(bestMachine->storeUri, storeParams); + sshStore->connect(); + storeUri = bestMachine->storeUri; + + } catch (std::exception & e) { + auto msg = chomp(drainFD(5, false)); + printError("cannot build on '%s': %s%s", + bestMachine->storeUri, e.what(), + (msg.empty() ? "" : ": " + msg)); + bestMachine->enabled = false; + continue; + } + + goto connected; + } + } + +connected: + close(5); + + std::cerr << "# accept\n" << storeUri << "\n"; + + auto inputs = readStrings(source); + auto outputs = readStrings(source); + + AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); + + auto old = signal(SIGALRM, handleAlarm); + alarm(15 * 60); + if (!lockFile(uploadLock.get(), ltWrite, true)) + printError("somebody is hogging the upload lock for '%s', continuing..."); + alarm(0); + signal(SIGALRM, old); + } + + auto substitute = settings.buildersUseSubstitutes ? Substitute : NoSubstitute; + + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying dependencies to '%s'", storeUri)); + copyPaths(store, ref(sshStore), inputs, NoRepair, NoCheckSigs, substitute); + } + + uploadLock = -1; + + BasicDerivation drv(readDerivation(store->realStoreDir + "/" + baseNameOf(drvPath))); + drv.inputSrcs = inputs; + + auto result = sshStore->buildDerivation(drvPath, drv); + + if (!result.success()) + throw Error("build of '%s' on '%s' failed: %s", drvPath, storeUri, result.errorMsg); + + PathSet missing; + for (auto & path : outputs) + if (!store->isValidPath(path)) missing.insert(path); + + if (!missing.empty()) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri)); + store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */ + copyPaths(ref(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute); + } + + return 0; + } +} + +static RegisterLegacyCommand s1("build-remote", _main); diff --git a/third_party/nix/src/cpptoml/LICENSE b/third_party/nix/src/cpptoml/LICENSE new file mode 100644 index 000000000..8802c4fa5 --- /dev/null +++ b/third_party/nix/src/cpptoml/LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2014 Chase Geigle + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/third_party/nix/src/cpptoml/cpptoml.h b/third_party/nix/src/cpptoml/cpptoml.h new file mode 100644 index 000000000..5a00da3b4 --- /dev/null +++ b/third_party/nix/src/cpptoml/cpptoml.h @@ -0,0 +1,3668 @@ +/** + * @file cpptoml.h + * @author Chase Geigle + * @date May 2013 + */ + +#ifndef CPPTOML_H +#define CPPTOML_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __cplusplus > 201103L +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(__clang__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason))) +#elif defined(__GNUG__) +#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated)) +#elif defined(_MSC_VER) +#if _MSC_VER < 1910 +#define CPPTOML_DEPRECATED(reason) __declspec(deprecated) +#else +#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] +#endif +#endif + +namespace cpptoml +{ +class writer; // forward declaration +class base; // forward declaration +#if defined(CPPTOML_USE_MAP) +// a std::map will ensure that entries a sorted, albeit at a slight +// performance penalty relative to the (default) unordered_map +using string_to_base_map = std::map>; +#else +// by default an unordered_map is used for best performance as the +// toml specification does not require entries to be sorted +using string_to_base_map + = std::unordered_map>; +#endif + +// if defined, `base` will retain type information in form of an enum class +// such that static_cast can be used instead of dynamic_cast +// #define CPPTOML_NO_RTTI + +template +class option +{ + public: + option() : empty_{true} + { + // nothing + } + + option(T value) : empty_{false}, value_(std::move(value)) + { + // nothing + } + + explicit operator bool() const + { + return !empty_; + } + + const T& operator*() const + { + return value_; + } + + const T* operator->() const + { + return &value_; + } + + template + T value_or(U&& alternative) const + { + if (!empty_) + return value_; + return static_cast(std::forward(alternative)); + } + + private: + bool empty_; + T value_; +}; + +struct local_date +{ + int year = 0; + int month = 0; + int day = 0; +}; + +struct local_time +{ + int hour = 0; + int minute = 0; + int second = 0; + int microsecond = 0; +}; + +struct zone_offset +{ + int hour_offset = 0; + int minute_offset = 0; +}; + +struct local_datetime : local_date, local_time +{ +}; + +struct offset_datetime : local_datetime, zone_offset +{ + static inline struct offset_datetime from_zoned(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + + char buf[16]; + strftime(buf, 16, "%z", &t); + + int offset = std::stoi(buf); + dt.hour_offset = offset / 100; + dt.minute_offset = offset % 100; + return dt; + } + + CPPTOML_DEPRECATED("from_local has been renamed to from_zoned") + static inline struct offset_datetime from_local(const struct tm& t) + { + return from_zoned(t); + } + + static inline struct offset_datetime from_utc(const struct tm& t) + { + offset_datetime dt; + dt.year = t.tm_year + 1900; + dt.month = t.tm_mon + 1; + dt.day = t.tm_mday; + dt.hour = t.tm_hour; + dt.minute = t.tm_min; + dt.second = t.tm_sec; + return dt; + } +}; + +CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime") +typedef offset_datetime datetime; + +class fill_guard +{ + public: + fill_guard(std::ostream& os) : os_(os), fill_{os.fill()} + { + // nothing + } + + ~fill_guard() + { + os_.fill(fill_); + } + + private: + std::ostream& os_; + std::ostream::char_type fill_; +}; + +inline std::ostream& operator<<(std::ostream& os, const local_date& dt) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2) + << dt.day; + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_time& ltime) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":" + << setw(2) << ltime.second; + + if (ltime.microsecond > 0) + { + os << "."; + int power = 100000; + for (int curr_us = ltime.microsecond; curr_us; power /= 10) + { + auto num = curr_us / power; + os << num; + curr_us -= num * power; + } + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo) +{ + fill_guard g{os}; + os.fill('0'); + + using std::setw; + + if (zo.hour_offset != 0 || zo.minute_offset != 0) + { + if (zo.hour_offset > 0) + { + os << "+"; + } + else + { + os << "-"; + } + os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2) + << std::abs(zo.minute_offset); + } + else + { + os << "Z"; + } + + return os; +} + +inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt) +{ + return os << static_cast(dt) << "T" + << static_cast(dt); +} + +inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt) +{ + return os << static_cast(dt) + << static_cast(dt); +} + +template +struct is_one_of; + +template +struct is_one_of : std::is_same +{ +}; + +template +struct is_one_of +{ + const static bool value + = std::is_same::value || is_one_of::value; +}; + +template +class value; + +template +struct valid_value + : is_one_of +{ +}; + +template +struct value_traits; + +template +struct valid_value_or_string_convertible +{ + + const static bool value = valid_value::type>::value + || std::is_convertible::value; +}; + +template +struct value_traits::value>::type> +{ + using value_type = typename std::conditional< + valid_value::type>::value, + typename std::decay::type, std::string>::type; + + using type = value; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template +struct value_traits< + T, + typename std::enable_if< + !valid_value_or_string_convertible::value + && std::is_floating_point::type>::value>::type> +{ + using value_type = typename std::decay::type; + + using type = value; + + static value_type construct(T&& val) + { + return value_type(val); + } +}; + +template +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible::value + && !std::is_floating_point::type>::value + && std::is_signed::type>::value>::type> +{ + using value_type = int64_t; + + using type = value; + + static value_type construct(T&& val) + { + if (val < (std::numeric_limits::min)()) + throw std::underflow_error{"constructed value cannot be " + "represented by a 64-bit signed " + "integer"}; + + if (val > (std::numeric_limits::max)()) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast(val); + } +}; + +template +struct value_traits< + T, typename std::enable_if< + !valid_value_or_string_convertible::value + && std::is_unsigned::type>::value>::type> +{ + using value_type = int64_t; + + using type = value; + + static value_type construct(T&& val) + { + if (val > static_cast((std::numeric_limits::max)())) + throw std::overflow_error{"constructed value cannot be represented " + "by a 64-bit signed integer"}; + + return static_cast(val); + } +}; + +class array; +class table; +class table_array; + +template +struct array_of_trait +{ + using return_type = option>; +}; + +template <> +struct array_of_trait +{ + using return_type = option>>; +}; + +template +inline std::shared_ptr::type> make_value(T&& val); +inline std::shared_ptr make_array(); + +namespace detail +{ +template +inline std::shared_ptr make_element(); +} + +inline std::shared_ptr make_table(); +inline std::shared_ptr make_table_array(bool is_inline = false); + +#if defined(CPPTOML_NO_RTTI) +/// Base type used to store underlying data type explicitly if RTTI is disabled +enum class base_type +{ + NONE, + STRING, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + TABLE, + ARRAY, + TABLE_ARRAY +}; + +/// Type traits class to convert C++ types to enum member +template +struct base_type_traits; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::STRING; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::LOCAL_TIME; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::LOCAL_DATE; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::LOCAL_DATETIME; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::OFFSET_DATETIME; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::INT; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::FLOAT; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::BOOL; +}; + +template <> +struct base_type_traits
+{ + static const base_type type = base_type::TABLE; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::ARRAY; +}; + +template <> +struct base_type_traits +{ + static const base_type type = base_type::TABLE_ARRAY; +}; +#endif + +/** + * A generic base TOML value used for type erasure. + */ +class base : public std::enable_shared_from_this +{ + public: + virtual ~base() = default; + + virtual std::shared_ptr clone() const = 0; + + /** + * Determines if the given TOML element is a value. + */ + virtual bool is_value() const + { + return false; + } + + /** + * Determines if the given TOML element is a table. + */ + virtual bool is_table() const + { + return false; + } + + /** + * Converts the TOML element into a table. + */ + std::shared_ptr
as_table() + { + if (is_table()) + return std::static_pointer_cast
(shared_from_this()); + return nullptr; + } + /** + * Determines if the TOML element is an array of "leaf" elements. + */ + virtual bool is_array() const + { + return false; + } + + /** + * Converts the TOML element to an array. + */ + std::shared_ptr as_array() + { + if (is_array()) + return std::static_pointer_cast(shared_from_this()); + return nullptr; + } + + /** + * Determines if the given TOML element is an array of tables. + */ + virtual bool is_table_array() const + { + return false; + } + + /** + * Converts the TOML element into a table array. + */ + std::shared_ptr as_table_array() + { + if (is_table_array()) + return std::static_pointer_cast(shared_from_this()); + return nullptr; + } + + /** + * Attempts to coerce the TOML element into a concrete TOML value + * of type T. + */ + template + std::shared_ptr> as(); + + template + std::shared_ptr> as() const; + + template + void accept(Visitor&& visitor, Args&&... args) const; + +#if defined(CPPTOML_NO_RTTI) + base_type type() const + { + return type_; + } + + protected: + base(const base_type t) : type_(t) + { + // nothing + } + + private: + const base_type type_ = base_type::NONE; + +#else + protected: + base() + { + // nothing + } +#endif +}; + +/** + * A concrete TOML value representing the "leaves" of the "tree". + */ +template +class value : public base +{ + struct make_shared_enabler + { + // nothing; this is a private key accessible only to friends + }; + + template + friend std::shared_ptr::type> + cpptoml::make_value(U&& val); + + public: + static_assert(valid_value::value, "invalid value type"); + + std::shared_ptr clone() const override; + + value(const make_shared_enabler&, const T& val) : value(val) + { + // nothing; note that users cannot actually invoke this function + // because they lack access to the make_shared_enabler. + } + + bool is_value() const override + { + return true; + } + + /** + * Gets the data associated with this value. + */ + T& get() + { + return data_; + } + + /** + * Gets the data associated with this value. Const version. + */ + const T& get() const + { + return data_; + } + + private: + T data_; + + /** + * Constructs a value from the given data. + */ +#if defined(CPPTOML_NO_RTTI) + value(const T& val) : base(base_type_traits::type), data_(val) + { + } +#else + value(const T& val) : data_(val) + { + } +#endif + + value(const value& val) = delete; + value& operator=(const value& val) = delete; +}; + +template +std::shared_ptr::type> make_value(T&& val) +{ + using value_type = typename value_traits::type; + using enabler = typename value_type::make_shared_enabler; + return std::make_shared( + enabler{}, value_traits::construct(std::forward(val))); +} + +template +inline std::shared_ptr> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits::type) + return std::static_pointer_cast>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast>(shared_from_this()); +#endif +} + +// special case value to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr> base::as() +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast>(shared_from_this()); + + if (type() == base_type::INT) + { + auto v = std::static_pointer_cast>(shared_from_this()); + return make_value(static_cast(v->get())); + } +#else + if (auto v = std::dynamic_pointer_cast>(shared_from_this())) + return v; + + if (auto v = std::dynamic_pointer_cast>(shared_from_this())) + return make_value(static_cast(v->get())); +#endif + + return nullptr; +} + +template +inline std::shared_ptr> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type_traits::type) + return std::static_pointer_cast>(shared_from_this()); + else + return nullptr; +#else + return std::dynamic_pointer_cast>(shared_from_this()); +#endif +} + +// special case value to allow getting an integer parameter as a +// double value +template <> +inline std::shared_ptr> base::as() const +{ +#if defined(CPPTOML_NO_RTTI) + if (type() == base_type::FLOAT) + return std::static_pointer_cast>( + shared_from_this()); + + if (type() == base_type::INT) + { + auto v = as(); + // the below has to be a non-const value due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value(static_cast(v->get())); + } +#else + if (auto v + = std::dynamic_pointer_cast>(shared_from_this())) + return v; + + if (auto v = as()) + { + // the below has to be a non-const value due to a bug in + // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 + return make_value(static_cast(v->get())); + } +#endif + + return nullptr; +} + +/** + * Exception class for array insertion errors. + */ +class array_exception : public std::runtime_error +{ + public: + array_exception(const std::string& err) : std::runtime_error{err} + { + } +}; + +class array : public base +{ + public: + friend std::shared_ptr make_array(); + + std::shared_ptr clone() const override; + + virtual bool is_array() const override + { + return true; + } + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector>::const_iterator; + + iterator begin() + { + return values_.begin(); + } + + const_iterator begin() const + { + return values_.begin(); + } + + iterator end() + { + return values_.end(); + } + + const_iterator end() const + { + return values_.end(); + } + + /** + * Obtains the array (vector) of base values. + */ + std::vector>& get() + { + return values_; + } + + /** + * Obtains the array (vector) of base values. Const version. + */ + const std::vector>& get() const + { + return values_; + } + + std::shared_ptr at(size_t idx) const + { + return values_.at(idx); + } + + /** + * Obtains an array of values. Note that elements may be + * nullptr if they cannot be converted to a value. + */ + template + std::vector>> array_of() const + { + std::vector>> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr v) { return v->as(); }); + + return result; + } + + /** + * Obtains a option>. The option will be empty if the array + * contains values that are not of type T. + */ + template + inline typename array_of_trait::return_type get_array_of() const + { + std::vector result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as()) + result.push_back(v->get()); + else + return {}; + } + + return {std::move(result)}; + } + + /** + * Obtains an array of arrays. Note that elements may be nullptr + * if they cannot be converted to a array. + */ + std::vector> nested_array() const + { + std::vector> result(values_.size()); + + std::transform(values_.begin(), values_.end(), result.begin(), + [&](std::shared_ptr v) -> std::shared_ptr { + if (v->is_array()) + return std::static_pointer_cast(v); + return std::shared_ptr{}; + }); + + return result; + } + + /** + * Add a value to the end of the array + */ + template + void push_back(const std::shared_ptr>& val) + { + if (values_.empty() || values_[0]->as()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Add an array to the end of the array + */ + void push_back(const std::shared_ptr& val) + { + if (values_.empty() || values_[0]->is_array()) + { + values_.push_back(val); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for adding a simple element to the end + * of the array. + */ + template + void push_back(T&& val, typename value_traits::type* = 0) + { + push_back(make_value(std::forward(val))); + } + + /** + * Insert a value into the array + */ + template + iterator insert(iterator position, const std::shared_ptr>& value) + { + if (values_.empty() || values_[0]->as()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Insert an array into the array + */ + iterator insert(iterator position, const std::shared_ptr& value) + { + if (values_.empty() || values_[0]->is_array()) + { + return values_.insert(position, value); + } + else + { + throw array_exception{"Arrays must be homogenous."}; + } + } + + /** + * Convenience function for inserting a simple element in the array + */ + template + iterator insert(iterator position, T&& val, + typename value_traits::type* = 0) + { + return insert(position, make_value(std::forward(val))); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return values_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + values_.clear(); + } + + /** + * Reserve space for n values. + */ + void reserve(size_type n) + { + values_.reserve(n); + } + + private: +#if defined(CPPTOML_NO_RTTI) + array() : base(base_type::ARRAY) + { + // empty + } +#else + array() = default; +#endif + + template + array(InputIterator begin, InputIterator end) : values_{begin, end} + { + // nothing + } + + array(const array& obj) = delete; + array& operator=(const array& obj) = delete; + + std::vector> values_; +}; + +inline std::shared_ptr make_array() +{ + struct make_shared_enabler : public array + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared(); +} + +namespace detail +{ +template <> +inline std::shared_ptr make_element() +{ + return make_array(); +} +} // namespace detail + +/** + * Obtains a option>. The option will be empty if the array + * contains values that are not of type T. + */ +template <> +inline typename array_of_trait::return_type +array::get_array_of() const +{ + std::vector> result; + result.reserve(values_.size()); + + for (const auto& val : values_) + { + if (auto v = val->as_array()) + result.push_back(v); + else + return {}; + } + + return {std::move(result)}; +} + +class table; + +class table_array : public base +{ + friend class table; + friend std::shared_ptr make_table_array(bool); + + public: + std::shared_ptr clone() const override; + + using size_type = std::size_t; + + /** + * arrays can be iterated over + */ + using iterator = std::vector>::iterator; + + /** + * arrays can be iterated over. Const version. + */ + using const_iterator = std::vector>::const_iterator; + + iterator begin() + { + return array_.begin(); + } + + const_iterator begin() const + { + return array_.begin(); + } + + iterator end() + { + return array_.end(); + } + + const_iterator end() const + { + return array_.end(); + } + + virtual bool is_table_array() const override + { + return true; + } + + std::vector>& get() + { + return array_; + } + + const std::vector>& get() const + { + return array_; + } + + /** + * Add a table to the end of the array + */ + void push_back(const std::shared_ptr
& val) + { + array_.push_back(val); + } + + /** + * Insert a table into the array + */ + iterator insert(iterator position, const std::shared_ptr
& value) + { + return array_.insert(position, value); + } + + /** + * Erase an element from the array + */ + iterator erase(iterator position) + { + return array_.erase(position); + } + + /** + * Clear the array + */ + void clear() + { + array_.clear(); + } + + /** + * Reserve space for n tables. + */ + void reserve(size_type n) + { + array_.reserve(n); + } + + /** + * Whether or not the table array is declared inline. This mostly + * matters for parsing, where statically defined arrays cannot be + * appended to using the array-of-table syntax. + */ + bool is_inline() const + { + return is_inline_; + } + + private: +#if defined(CPPTOML_NO_RTTI) + table_array(bool is_inline = false) + : base(base_type::TABLE_ARRAY), is_inline_(is_inline) + { + // nothing + } +#else + table_array(bool is_inline = false) : is_inline_(is_inline) + { + // nothing + } +#endif + + table_array(const table_array& obj) = delete; + table_array& operator=(const table_array& rhs) = delete; + + std::vector> array_; + const bool is_inline_ = false; +}; + +inline std::shared_ptr make_table_array(bool is_inline) +{ + struct make_shared_enabler : public table_array + { + make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) + { + // nothing + } + }; + + return std::make_shared(is_inline); +} + +namespace detail +{ +template <> +inline std::shared_ptr make_element() +{ + return make_table_array(true); +} +} // namespace detail + +// The below are overloads for fetching specific value types out of a value +// where special casting behavior (like bounds checking) is desired + +template +typename std::enable_if::value + && std::is_signed::value, + option>::type +get_impl(const std::shared_ptr& elem) +{ + if (auto v = elem->as()) + { + if (v->get() < (std::numeric_limits::min)()) + throw std::underflow_error{ + "T cannot represent the value requested in get"}; + + if (v->get() > (std::numeric_limits::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast(v->get())}; + } + else + { + return {}; + } +} + +template +typename std::enable_if::value + && std::is_unsigned::value, + option>::type +get_impl(const std::shared_ptr& elem) +{ + if (auto v = elem->as()) + { + if (v->get() < 0) + throw std::underflow_error{"T cannot store negative value in get"}; + + if (static_cast(v->get()) > (std::numeric_limits::max)()) + throw std::overflow_error{ + "T cannot represent the value requested in get"}; + + return {static_cast(v->get())}; + } + else + { + return {}; + } +} + +template +typename std::enable_if::value + || std::is_same::value, + option>::type +get_impl(const std::shared_ptr& elem) +{ + if (auto v = elem->as()) + { + return {v->get()}; + } + else + { + return {}; + } +} + +/** + * Represents a TOML keytable. + */ +class table : public base +{ + public: + friend class table_array; + friend std::shared_ptr
make_table(); + + std::shared_ptr clone() const override; + + /** + * tables can be iterated over. + */ + using iterator = string_to_base_map::iterator; + + /** + * tables can be iterated over. Const version. + */ + using const_iterator = string_to_base_map::const_iterator; + + iterator begin() + { + return map_.begin(); + } + + const_iterator begin() const + { + return map_.begin(); + } + + iterator end() + { + return map_.end(); + } + + const_iterator end() const + { + return map_.end(); + } + + bool is_table() const override + { + return true; + } + + bool empty() const + { + return map_.empty(); + } + + /** + * Determines if this key table contains the given key. + */ + bool contains(const std::string& key) const + { + return map_.find(key) != map_.end(); + } + + /** + * Determines if this key table contains the given key. Will + * resolve "qualified keys". Qualified keys are the full access + * path separated with dots like "grandparent.parent.child". + */ + bool contains_qualified(const std::string& key) const + { + return resolve_qualified(key); + } + + /** + * Obtains the base for a given key. + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr get(const std::string& key) const + { + return map_.at(key); + } + + /** + * Obtains the base for a given key. Will resolve "qualified + * keys". Qualified keys are the full access path separated with + * dots like "grandparent.parent.child". + * + * @throw std::out_of_range if the key does not exist + */ + std::shared_ptr get_qualified(const std::string& key) const + { + std::shared_ptr p; + resolve_qualified(key, &p); + return p; + } + + /** + * Obtains a table for a given key, if possible. + */ + std::shared_ptr
get_table(const std::string& key) const + { + if (contains(key) && get(key)->is_table()) + return std::static_pointer_cast
(get(key)); + return nullptr; + } + + /** + * Obtains a table for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr
get_table_qualified(const std::string& key) const + { + if (contains_qualified(key) && get_qualified(key)->is_table()) + return std::static_pointer_cast
(get_qualified(key)); + return nullptr; + } + + /** + * Obtains an array for a given key. + */ + std::shared_ptr get_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_array(); + } + + /** + * Obtains an array for a given key. Will resolve "qualified keys". + */ + std::shared_ptr get_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_array(); + } + + /** + * Obtains a table_array for a given key, if possible. + */ + std::shared_ptr get_table_array(const std::string& key) const + { + if (!contains(key)) + return nullptr; + return get(key)->as_table_array(); + } + + /** + * Obtains a table_array for a given key, if possible. Will resolve + * "qualified keys". + */ + std::shared_ptr + get_table_array_qualified(const std::string& key) const + { + if (!contains_qualified(key)) + return nullptr; + return get_qualified(key)->as_table_array(); + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. + */ + template + option get_as(const std::string& key) const + { + try + { + return get_impl(get(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get a value corresponding + * to the template parameter from a given key. Will resolve "qualified + * keys". + */ + template + option get_qualified_as(const std::string& key) const + { + try + { + return get_impl(get_qualified(key)); + } + catch (const std::out_of_range&) + { + return {}; + } + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template + inline typename array_of_trait::return_type + get_array_of(const std::string& key) const + { + if (auto v = get_array(key)) + { + std::vector result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Helper function that attempts to get an array of values of a given + * type corresponding to the template parameter for a given key. Will + * resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ + template + inline typename array_of_trait::return_type + get_qualified_array_of(const std::string& key) const + { + if (auto v = get_array_qualified(key)) + { + std::vector result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as()) + result.push_back(val->get()); + else + return {}; + } + return {std::move(result)}; + } + + return {}; + } + + /** + * Adds an element to the keytable. + */ + void insert(const std::string& key, const std::shared_ptr& value) + { + map_[key] = value; + } + + /** + * Convenience shorthand for adding a simple element to the + * keytable. + */ + template + void insert(const std::string& key, T&& val, + typename value_traits::type* = 0) + { + insert(key, make_value(std::forward(val))); + } + + /** + * Removes an element from the table. + */ + void erase(const std::string& key) + { + map_.erase(key); + } + + private: +#if defined(CPPTOML_NO_RTTI) + table() : base(base_type::TABLE) + { + // nothing + } +#else + table() + { + // nothing + } +#endif + + table(const table& obj) = delete; + table& operator=(const table& rhs) = delete; + + std::vector split(const std::string& value, + char separator) const + { + std::vector result; + std::string::size_type p = 0; + std::string::size_type q; + while ((q = value.find(separator, p)) != std::string::npos) + { + result.emplace_back(value, p, q - p); + p = q + 1; + } + result.emplace_back(value, p); + return result; + } + + // If output parameter p is specified, fill it with the pointer to the + // specified entry and throw std::out_of_range if it couldn't be found. + // + // Otherwise, just return true if the entry could be found or false + // otherwise and do not throw. + bool resolve_qualified(const std::string& key, + std::shared_ptr* p = nullptr) const + { + auto parts = split(key, '.'); + auto last_key = parts.back(); + parts.pop_back(); + + auto cur_table = this; + for (const auto& part : parts) + { + cur_table = cur_table->get_table(part).get(); + if (!cur_table) + { + if (!p) + return false; + + throw std::out_of_range{key + " is not a valid key"}; + } + } + + if (!p) + return cur_table->map_.count(last_key) != 0; + + *p = cur_table->map_.at(last_key); + return true; + } + + string_to_base_map map_; +}; + +/** + * Helper function that attempts to get an array of arrays for a given + * key. + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait::return_type +table::get_array_of(const std::string& key) const +{ + if (auto v = get_array(key)) + { + std::vector> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +/** + * Helper function that attempts to get an array of arrays for a given + * key. Will resolve "qualified keys". + * + * If the key doesn't exist, doesn't exist as an array type, or one or + * more keys inside the array type are not of type T, an empty option + * is returned. Otherwise, an option containing a vector of the values + * is returned. + */ +template <> +inline typename array_of_trait::return_type +table::get_qualified_array_of(const std::string& key) const +{ + if (auto v = get_array_qualified(key)) + { + std::vector> result; + result.reserve(v->get().size()); + + for (const auto& b : v->get()) + { + if (auto val = b->as_array()) + result.push_back(val); + else + return {}; + } + + return {std::move(result)}; + } + + return {}; +} + +std::shared_ptr
make_table() +{ + struct make_shared_enabler : public table + { + make_shared_enabler() + { + // nothing + } + }; + + return std::make_shared(); +} + +namespace detail +{ +template <> +inline std::shared_ptr
make_element
() +{ + return make_table(); +} +} // namespace detail + +template +std::shared_ptr value::clone() const +{ + return make_value(data_); +} + +inline std::shared_ptr array::clone() const +{ + auto result = make_array(); + result->reserve(values_.size()); + for (const auto& ptr : values_) + result->values_.push_back(ptr->clone()); + return result; +} + +inline std::shared_ptr table_array::clone() const +{ + auto result = make_table_array(is_inline()); + result->reserve(array_.size()); + for (const auto& ptr : array_) + result->array_.push_back(ptr->clone()->as_table()); + return result; +} + +inline std::shared_ptr table::clone() const +{ + auto result = make_table(); + for (const auto& pr : map_) + result->insert(pr.first, pr.second->clone()); + return result; +} + +/** + * Exception class for all TOML parsing errors. + */ +class parse_exception : public std::runtime_error +{ + public: + parse_exception(const std::string& err) : std::runtime_error{err} + { + } + + parse_exception(const std::string& err, std::size_t line_number) + : std::runtime_error{err + " at line " + std::to_string(line_number)} + { + } +}; + +inline bool is_number(char c) +{ + return c >= '0' && c <= '9'; +} + +inline bool is_hex(char c) +{ + return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); +} + +/** + * Helper object for consuming expected characters. + */ +template +class consumer +{ + public: + consumer(std::string::iterator& it, const std::string::iterator& end, + OnError&& on_error) + : it_(it), end_(end), on_error_(std::forward(on_error)) + { + // nothing + } + + void operator()(char c) + { + if (it_ == end_ || *it_ != c) + on_error_(); + ++it_; + } + + template + void operator()(const char (&str)[N]) + { + std::for_each(std::begin(str), std::end(str) - 1, + [&](char c) { (*this)(c); }); + } + + void eat_or(char a, char b) + { + if (it_ == end_ || (*it_ != a && *it_ != b)) + on_error_(); + ++it_; + } + + int eat_digits(int len) + { + int val = 0; + for (int i = 0; i < len; ++i) + { + if (!is_number(*it_) || it_ == end_) + on_error_(); + val = 10 * val + (*it_++ - '0'); + } + return val; + } + + void error() + { + on_error_(); + } + + private: + std::string::iterator& it_; + const std::string::iterator& end_; + OnError on_error_; +}; + +template +consumer make_consumer(std::string::iterator& it, + const std::string::iterator& end, + OnError&& on_error) +{ + return consumer(it, end, std::forward(on_error)); +} + +// replacement for std::getline to handle incorrectly line-ended files +// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf +namespace detail +{ +inline std::istream& getline(std::istream& input, std::string& line) +{ + line.clear(); + + std::istream::sentry sentry{input, true}; + auto sb = input.rdbuf(); + + while (true) + { + auto c = sb->sbumpc(); + if (c == '\r') + { + if (sb->sgetc() == '\n') + c = sb->sbumpc(); + } + + if (c == '\n') + return input; + + if (c == std::istream::traits_type::eof()) + { + if (line.empty()) + input.setstate(std::ios::eofbit); + return input; + } + + line.push_back(static_cast(c)); + } +} +} // namespace detail + +/** + * The parser class. + */ +class parser +{ + public: + /** + * Parsers are constructed from streams. + */ + parser(std::istream& stream) : input_(stream) + { + // nothing + } + + parser& operator=(const parser& parser) = delete; + + /** + * Parses the stream this parser was created on until EOF. + * @throw parse_exception if there are errors in parsing + */ + std::shared_ptr
parse() + { + std::shared_ptr
root = make_table(); + + table* curr_table = root.get(); + + while (detail::getline(input_, line_)) + { + line_number_++; + auto it = line_.begin(); + auto end = line_.end(); + consume_whitespace(it, end); + if (it == end || *it == '#') + continue; + if (*it == '[') + { + curr_table = root.get(); + parse_table(it, end, curr_table); + } + else + { + parse_key_value(it, end, curr_table); + consume_whitespace(it, end); + eol_or_comment(it, end); + } + } + return root; + } + + private: +#if defined _MSC_VER + __declspec(noreturn) +#elif defined __GNUC__ + __attribute__((noreturn)) +#endif + void throw_parse_exception(const std::string& err) + { + throw parse_exception{err, line_number_}; + } + + void parse_table(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + // remove the beginning keytable marker + ++it; + if (it == end) + throw_parse_exception("Unexpected end of table"); + if (*it == '[') + parse_table_array(it, end, curr_table); + else + parse_single_table(it, end, curr_table); + } + + void parse_single_table(std::string::iterator& it, + const std::string::iterator& end, + table*& curr_table) + { + if (it == end || *it == ']') + throw_parse_exception("Table name cannot be empty"); + + std::string full_table_name; + bool inserted = false; + + auto key_end = [](char c) { return c == ']'; }; + + auto key_part_handler = [&](const std::string& part) { + if (part.empty()) + throw_parse_exception("Empty component of table name"); + + if (!full_table_name.empty()) + full_table_name += '.'; + full_table_name += part; + + if (curr_table->contains(part)) + { +#if !defined(__PGI) + auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr b = curr_table->get(part); +#endif + if (b->is_table()) + curr_table = static_cast(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_table_name + + "already exists as a value"); + } + else + { + inserted = true; + curr_table->insert(part, make_table()); + curr_table = static_cast(curr_table->get(part).get()); + } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); + + if (it == end) + throw_parse_exception( + "Unterminated table declaration; did you forget a ']'?"); + + if (*it != ']') + { + std::string errmsg{"Unexpected character in table definition: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + // table already existed + if (!inserted) + { + auto is_value + = [](const std::pair&>& p) { + return p.second->is_value(); + }; + + // if there are any values, we can't add values to this table + // since it has already been defined. If there aren't any + // values, then it was implicitly created by something like + // [a.b] + if (curr_table->empty() + || std::any_of(curr_table->begin(), curr_table->end(), + is_value)) + { + throw_parse_exception("Redefinition of table " + + full_table_name); + } + } + + ++it; + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_table_array(std::string::iterator& it, + const std::string::iterator& end, table*& curr_table) + { + ++it; + if (it == end || *it == ']') + throw_parse_exception("Table array name cannot be empty"); + + auto key_end = [](char c) { return c == ']'; }; + + std::string full_ta_name; + auto key_part_handler = [&](const std::string& part) { + if (part.empty()) + throw_parse_exception("Empty component of table array name"); + + if (!full_ta_name.empty()) + full_ta_name += '.'; + full_ta_name += part; + + if (curr_table->contains(part)) + { +#if !defined(__PGI) + auto b = curr_table->get(part); +#else + // Workaround for PGI compiler + std::shared_ptr b = curr_table->get(part); +#endif + + // if this is the end of the table array name, add an + // element to the table array that we just looked up, + // provided it was not declared inline + if (it != end && *it == ']') + { + if (!b->is_table_array()) + { + throw_parse_exception("Key " + full_ta_name + + " is not a table array"); + } + + auto v = b->as_table_array(); + + if (v->is_inline()) + { + throw_parse_exception("Static array " + full_ta_name + + " cannot be appended to"); + } + + v->get().push_back(make_table()); + curr_table = v->get().back().get(); + } + // otherwise, just keep traversing down the key name + else + { + if (b->is_table()) + curr_table = static_cast(b.get()); + else if (b->is_table_array()) + curr_table = std::static_pointer_cast(b) + ->get() + .back() + .get(); + else + throw_parse_exception("Key " + full_ta_name + + " already exists as a value"); + } + } + else + { + // if this is the end of the table array name, add a new + // table array and a new table inside that array for us to + // add keys to next + if (it != end && *it == ']') + { + curr_table->insert(part, make_table_array()); + auto arr = std::static_pointer_cast( + curr_table->get(part)); + arr->get().push_back(make_table()); + curr_table = arr->get().back().get(); + } + // otherwise, create the implicitly defined table and move + // down to it + else + { + curr_table->insert(part, make_table()); + curr_table + = static_cast(curr_table->get(part).get()); + } + } + }; + + key_part_handler(parse_key(it, end, key_end, key_part_handler)); + + // consume the last "]]" + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Unterminated table array name"); + }); + eat(']'); + eat(']'); + + consume_whitespace(it, end); + eol_or_comment(it, end); + } + + void parse_key_value(std::string::iterator& it, std::string::iterator& end, + table* curr_table) + { + auto key_end = [](char c) { return c == '='; }; + + auto key_part_handler = [&](const std::string& part) { + // two cases: this key part exists already, in which case it must + // be a table, or it doesn't exist in which case we must create + // an implicitly defined table + if (curr_table->contains(part)) + { + auto val = curr_table->get(part); + if (val->is_table()) + { + curr_table = static_cast(val.get()); + } + else + { + throw_parse_exception("Key " + part + + " already exists as a value"); + } + } + else + { + auto newtable = make_table(); + curr_table->insert(part, newtable); + curr_table = newtable.get(); + } + }; + + auto key = parse_key(it, end, key_end, key_part_handler); + + if (curr_table->contains(key)) + throw_parse_exception("Key " + key + " already present"); + if (it == end || *it != '=') + throw_parse_exception("Value must follow after a '='"); + ++it; + consume_whitespace(it, end); + curr_table->insert(key, parse_value(it, end)); + consume_whitespace(it, end); + } + + template + std::string + parse_key(std::string::iterator& it, const std::string::iterator& end, + KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) + { + // parse the key as a series of one or more simple-keys joined with '.' + while (it != end && !key_end(*it)) + { + auto part = parse_simple_key(it, end); + consume_whitespace(it, end); + + if (it == end || key_end(*it)) + { + return part; + } + + if (*it != '.') + { + std::string errmsg{"Unexpected character in key: "}; + errmsg += '"'; + errmsg += *it; + errmsg += '"'; + throw_parse_exception(errmsg); + } + + key_part_handler(part); + + // consume the dot + ++it; + } + + throw_parse_exception("Unexpected end of key"); + } + + std::string parse_simple_key(std::string::iterator& it, + const std::string::iterator& end) + { + consume_whitespace(it, end); + + if (it == end) + throw_parse_exception("Unexpected end of key (blank key?)"); + + if (*it == '"' || *it == '\'') + { + return string_literal(it, end, *it); + } + else + { + auto bke = std::find_if(it, end, [](char c) { + return c == '.' || c == '=' || c == ']'; + }); + return parse_bare_key(it, bke); + } + } + + std::string parse_bare_key(std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Bare key missing name"); + } + + auto key_end = end; + --key_end; + consume_backwards_whitespace(key_end, it); + ++key_end; + std::string key{it, key_end}; + + if (std::find(it, key_end, '#') != key_end) + { + throw_parse_exception("Bare key " + key + " cannot contain #"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == ' ' || c == '\t'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain whitespace"); + } + + if (std::find_if(it, key_end, + [](char c) { return c == '[' || c == ']'; }) + != key_end) + { + throw_parse_exception("Bare key " + key + + " cannot contain '[' or ']'"); + } + + it = end; + return key; + } + + enum class parse_type + { + STRING = 1, + LOCAL_TIME, + LOCAL_DATE, + LOCAL_DATETIME, + OFFSET_DATETIME, + INT, + FLOAT, + BOOL, + ARRAY, + INLINE_TABLE + }; + + std::shared_ptr parse_value(std::string::iterator& it, + std::string::iterator& end) + { + parse_type type = determine_value_type(it, end); + switch (type) + { + case parse_type::STRING: + return parse_string(it, end); + case parse_type::LOCAL_TIME: + return parse_time(it, end); + case parse_type::LOCAL_DATE: + case parse_type::LOCAL_DATETIME: + case parse_type::OFFSET_DATETIME: + return parse_date(it, end); + case parse_type::INT: + case parse_type::FLOAT: + return parse_number(it, end); + case parse_type::BOOL: + return parse_bool(it, end); + case parse_type::ARRAY: + return parse_array(it, end); + case parse_type::INLINE_TABLE: + return parse_inline_table(it, end); + default: + throw_parse_exception("Failed to parse value"); + } + } + + parse_type determine_value_type(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it == end) + { + throw_parse_exception("Failed to parse value type"); + } + if (*it == '"' || *it == '\'') + { + return parse_type::STRING; + } + else if (is_time(it, end)) + { + return parse_type::LOCAL_TIME; + } + else if (auto dtype = date_type(it, end)) + { + return *dtype; + } + else if (is_number(*it) || *it == '-' || *it == '+' + || (*it == 'i' && it + 1 != end && it[1] == 'n' + && it + 2 != end && it[2] == 'f') + || (*it == 'n' && it + 1 != end && it[1] == 'a' + && it + 2 != end && it[2] == 'n')) + { + return determine_number_type(it, end); + } + else if (*it == 't' || *it == 'f') + { + return parse_type::BOOL; + } + else if (*it == '[') + { + return parse_type::ARRAY; + } + else if (*it == '{') + { + return parse_type::INLINE_TABLE; + } + throw_parse_exception("Failed to parse value type"); + } + + parse_type determine_number_type(const std::string::iterator& it, + const std::string::iterator& end) + { + // determine if we are an integer or a float + auto check_it = it; + if (*check_it == '-' || *check_it == '+') + ++check_it; + + if (check_it == end) + throw_parse_exception("Malformed number"); + + if (*check_it == 'i' || *check_it == 'n') + return parse_type::FLOAT; + + while (check_it != end && is_number(*check_it)) + ++check_it; + if (check_it != end && *check_it == '.') + { + ++check_it; + while (check_it != end && is_number(*check_it)) + ++check_it; + return parse_type::FLOAT; + } + else + { + return parse_type::INT; + } + } + + std::shared_ptr> parse_string(std::string::iterator& it, + std::string::iterator& end) + { + auto delim = *it; + assert(delim == '"' || delim == '\''); + + // end is non-const here because we have to be able to potentially + // parse multiple lines in a string, not just one + auto check_it = it; + ++check_it; + if (check_it != end && *check_it == delim) + { + ++check_it; + if (check_it != end && *check_it == delim) + { + it = ++check_it; + return parse_multiline_string(it, end, delim); + } + } + return make_value(string_literal(it, end, delim)); + } + + std::shared_ptr> + parse_multiline_string(std::string::iterator& it, + std::string::iterator& end, char delim) + { + std::stringstream ss; + + auto is_ws = [](char c) { return c == ' ' || c == '\t'; }; + + bool consuming = false; + std::shared_ptr> ret; + + auto handle_line = [&](std::string::iterator& local_it, + std::string::iterator& local_end) { + if (consuming) + { + local_it = std::find_if_not(local_it, local_end, is_ws); + + // whole line is whitespace + if (local_it == local_end) + return; + } + + consuming = false; + + while (local_it != local_end) + { + // handle escaped characters + if (delim == '"' && *local_it == '\\') + { + auto check = local_it; + // check if this is an actual escape sequence or a + // whitespace escaping backslash + ++check; + consume_whitespace(check, local_end); + if (check == local_end) + { + consuming = true; + break; + } + + ss << parse_escape_code(local_it, local_end); + continue; + } + + // if we can end the string + if (std::distance(local_it, local_end) >= 3) + { + auto check = local_it; + // check for """ + if (*check++ == delim && *check++ == delim + && *check++ == delim) + { + local_it = check; + ret = make_value(ss.str()); + break; + } + } + + ss << *local_it++; + } + }; + + // handle the remainder of the current line + handle_line(it, end); + if (ret) + return ret; + + // start eating lines + while (detail::getline(input_, line_)) + { + ++line_number_; + + it = line_.begin(); + end = line_.end(); + + handle_line(it, end); + + if (ret) + return ret; + + if (!consuming) + ss << std::endl; + } + + throw_parse_exception("Unterminated multi-line basic string"); + } + + std::string string_literal(std::string::iterator& it, + const std::string::iterator& end, char delim) + { + ++it; + std::string val; + while (it != end) + { + // handle escaped characters + if (delim == '"' && *it == '\\') + { + val += parse_escape_code(it, end); + } + else if (*it == delim) + { + ++it; + consume_whitespace(it, end); + return val; + } + else + { + val += *it++; + } + } + throw_parse_exception("Unterminated string literal"); + } + + std::string parse_escape_code(std::string::iterator& it, + const std::string::iterator& end) + { + ++it; + if (it == end) + throw_parse_exception("Invalid escape sequence"); + char value; + if (*it == 'b') + { + value = '\b'; + } + else if (*it == 't') + { + value = '\t'; + } + else if (*it == 'n') + { + value = '\n'; + } + else if (*it == 'f') + { + value = '\f'; + } + else if (*it == 'r') + { + value = '\r'; + } + else if (*it == '"') + { + value = '"'; + } + else if (*it == '\\') + { + value = '\\'; + } + else if (*it == 'u' || *it == 'U') + { + return parse_unicode(it, end); + } + else + { + throw_parse_exception("Invalid escape sequence"); + } + ++it; + return std::string(1, value); + } + + std::string parse_unicode(std::string::iterator& it, + const std::string::iterator& end) + { + bool large = *it++ == 'U'; + auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000); + + if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) + { + throw_parse_exception( + "Unicode escape sequence is not a Unicode scalar value"); + } + + std::string result; + // See Table 3-6 of the Unicode standard + if (codepoint <= 0x7f) + { + // 1-byte codepoints: 00000000 0xxxxxxx + // repr: 0xxxxxxx + result += static_cast(codepoint & 0x7f); + } + else if (codepoint <= 0x7ff) + { + // 2-byte codepoints: 00000yyy yyxxxxxx + // repr: 110yyyyy 10xxxxxx + // + // 0x1f = 00011111 + // 0xc0 = 11000000 + // + result += static_cast(0xc0 | ((codepoint >> 6) & 0x1f)); + // + // 0x80 = 10000000 + // 0x3f = 00111111 + // + result += static_cast(0x80 | (codepoint & 0x3f)); + } + else if (codepoint <= 0xffff) + { + // 3-byte codepoints: zzzzyyyy yyxxxxxx + // repr: 1110zzzz 10yyyyyy 10xxxxxx + // + // 0xe0 = 11100000 + // 0x0f = 00001111 + // + result += static_cast(0xe0 | ((codepoint >> 12) & 0x0f)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x1f)); + result += static_cast(0x80 | (codepoint & 0x3f)); + } + else + { + // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx + // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx + // + // 0xf0 = 11110000 + // 0x07 = 00000111 + // + result += static_cast(0xf0 | ((codepoint >> 18) & 0x07)); + result += static_cast(0x80 | ((codepoint >> 12) & 0x3f)); + result += static_cast(0x80 | ((codepoint >> 6) & 0x3f)); + result += static_cast(0x80 | (codepoint & 0x3f)); + } + return result; + } + + uint32_t parse_hex(std::string::iterator& it, + const std::string::iterator& end, uint32_t place) + { + uint32_t value = 0; + while (place > 0) + { + if (it == end) + throw_parse_exception("Unexpected end of unicode sequence"); + + if (!is_hex(*it)) + throw_parse_exception("Invalid unicode escape sequence"); + + value += place * hex_to_digit(*it++); + place /= 16; + } + return value; + } + + uint32_t hex_to_digit(char c) + { + if (is_number(c)) + return static_cast(c - '0'); + return 10 + + static_cast(c + - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); + } + + std::shared_ptr parse_number(std::string::iterator& it, + const std::string::iterator& end) + { + auto check_it = it; + auto check_end = find_end_of_number(it, end); + + auto eat_sign = [&]() { + if (check_it != end && (*check_it == '-' || *check_it == '+')) + ++check_it; + }; + + auto check_no_leading_zero = [&]() { + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && check_it[1] != '.') + { + throw_parse_exception("Numbers may not have leading zeros"); + } + }; + + auto eat_digits = [&](bool (*check_char)(char)) { + auto beg = check_it; + while (check_it != end && check_char(*check_it)) + { + ++check_it; + if (check_it != end && *check_it == '_') + { + ++check_it; + if (check_it == end || !check_char(*check_it)) + throw_parse_exception("Malformed number"); + } + } + + if (check_it == beg) + throw_parse_exception("Malformed number"); + }; + + auto eat_hex = [&]() { eat_digits(&is_hex); }; + + auto eat_numbers = [&]() { eat_digits(&is_number); }; + + if (check_it != end && *check_it == '0' && check_it + 1 != check_end + && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) + { + ++check_it; + char base = *check_it; + ++check_it; + if (base == 'x') + { + eat_hex(); + return parse_int(it, check_it, 16); + } + else if (base == 'o') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 8, "0"); + it = start; + return val; + } + else // if (base == 'b') + { + auto start = check_it; + eat_numbers(); + auto val = parse_int(start, check_it, 2); + it = start; + return val; + } + } + + eat_sign(); + check_no_leading_zero(); + + if (check_it != end && check_it + 1 != end && check_it + 2 != end) + { + if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') + { + auto val = std::numeric_limits::infinity(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + else if (check_it[0] == 'n' && check_it[1] == 'a' + && check_it[2] == 'n') + { + auto val = std::numeric_limits::quiet_NaN(); + if (*it == '-') + val = -val; + it = check_it + 3; + return make_value(val); + } + } + + eat_numbers(); + + if (check_it != end + && (*check_it == '.' || *check_it == 'e' || *check_it == 'E')) + { + bool is_exp = *check_it == 'e' || *check_it == 'E'; + + ++check_it; + if (check_it == end) + throw_parse_exception("Floats must have trailing digits"); + + auto eat_exp = [&]() { + eat_sign(); + check_no_leading_zero(); + eat_numbers(); + }; + + if (is_exp) + eat_exp(); + else + eat_numbers(); + + if (!is_exp && check_it != end + && (*check_it == 'e' || *check_it == 'E')) + { + ++check_it; + eat_exp(); + } + + return parse_float(it, check_it); + } + else + { + return parse_int(it, check_it); + } + } + + std::shared_ptr> parse_int(std::string::iterator& it, + const std::string::iterator& end, + int base = 10, + const char* prefix = "") + { + std::string v{it, end}; + v = prefix + v; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + try + { + return make_value(std::stoll(v, nullptr, base)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr> parse_float(std::string::iterator& it, + const std::string::iterator& end) + { + std::string v{it, end}; + v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); + it = end; + char decimal_point = std::localeconv()->decimal_point[0]; + std::replace(v.begin(), v.end(), '.', decimal_point); + try + { + return make_value(std::stod(v)); + } + catch (const std::invalid_argument& ex) + { + throw_parse_exception("Malformed number (invalid argument: " + + std::string{ex.what()} + ")"); + } + catch (const std::out_of_range& ex) + { + throw_parse_exception("Malformed number (out of range: " + + std::string{ex.what()} + ")"); + } + } + + std::shared_ptr> parse_bool(std::string::iterator& it, + const std::string::iterator& end) + { + auto eat = make_consumer(it, end, [this]() { + throw_parse_exception("Attempted to parse invalid boolean value"); + }); + + if (*it == 't') + { + eat("true"); + return make_value(true); + } + else if (*it == 'f') + { + eat("false"); + return make_value(false); + } + + eat.error(); + return nullptr; + } + + std::string::iterator find_end_of_number(std::string::iterator it, + std::string::iterator end) + { + auto ret = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' + && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; + }); + if (ret != end && ret + 1 != end && ret + 2 != end) + { + if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') + || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) + { + ret = ret + 3; + } + } + return ret; + } + + std::string::iterator find_end_of_date(std::string::iterator it, + std::string::iterator end) + { + auto end_of_date = std::find_if(it, end, [](char c) { + return !is_number(c) && c != '-'; + }); + if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end + && is_number(end_of_date[1])) + end_of_date++; + return std::find_if(end_of_date, end, [](char c) { + return !is_number(c) && c != 'T' && c != 'Z' && c != ':' + && c != '-' && c != '+' && c != '.'; + }); + } + + std::string::iterator find_end_of_time(std::string::iterator it, + std::string::iterator end) + { + return std::find_if(it, end, [](char c) { + return !is_number(c) && c != ':' && c != '.'; + }); + } + + local_time read_time(std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + + auto eat = make_consumer( + it, time_end, [&]() { throw_parse_exception("Malformed time"); }); + + local_time ltime; + + ltime.hour = eat.eat_digits(2); + eat(':'); + ltime.minute = eat.eat_digits(2); + eat(':'); + ltime.second = eat.eat_digits(2); + + int power = 100000; + if (it != time_end && *it == '.') + { + ++it; + while (it != time_end && is_number(*it)) + { + ltime.microsecond += power * (*it++ - '0'); + power /= 10; + } + } + + if (it != time_end) + throw_parse_exception("Malformed time"); + + return ltime; + } + + std::shared_ptr> + parse_time(std::string::iterator& it, const std::string::iterator& end) + { + return make_value(read_time(it, end)); + } + + std::shared_ptr parse_date(std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + + auto eat = make_consumer( + it, date_end, [&]() { throw_parse_exception("Malformed date"); }); + + local_date ldate; + ldate.year = eat.eat_digits(4); + eat('-'); + ldate.month = eat.eat_digits(2); + eat('-'); + ldate.day = eat.eat_digits(2); + + if (it == date_end) + return make_value(ldate); + + eat.eat_or('T', ' '); + + local_datetime ldt; + static_cast(ldt) = ldate; + static_cast(ldt) = read_time(it, date_end); + + if (it == date_end) + return make_value(ldt); + + offset_datetime dt; + static_cast(dt) = ldt; + + int hoff = 0; + int moff = 0; + if (*it == '+' || *it == '-') + { + auto plus = *it == '+'; + ++it; + + hoff = eat.eat_digits(2); + dt.hour_offset = (plus) ? hoff : -hoff; + eat(':'); + moff = eat.eat_digits(2); + dt.minute_offset = (plus) ? moff : -moff; + } + else if (*it == 'Z') + { + ++it; + } + + if (it != date_end) + throw_parse_exception("Malformed date"); + + return make_value(dt); + } + + std::shared_ptr parse_array(std::string::iterator& it, + std::string::iterator& end) + { + // this gets ugly because of the "homogeneity" restriction: + // arrays can either be of only one type, or contain arrays + // (each of those arrays could be of different types, though) + // + // because of the latter portion, we don't really have a choice + // but to represent them as arrays of base values... + ++it; + + // ugh---have to read the first value to determine array type... + skip_whitespace_and_comments(it, end); + + // edge case---empty array + if (*it == ']') + { + ++it; + return make_array(); + } + + auto val_end = std::find_if( + it, end, [](char c) { return c == ',' || c == ']' || c == '#'; }); + parse_type type = determine_value_type(it, val_end); + switch (type) + { + case parse_type::STRING: + return parse_value_array(it, end); + case parse_type::LOCAL_TIME: + return parse_value_array(it, end); + case parse_type::LOCAL_DATE: + return parse_value_array(it, end); + case parse_type::LOCAL_DATETIME: + return parse_value_array(it, end); + case parse_type::OFFSET_DATETIME: + return parse_value_array(it, end); + case parse_type::INT: + return parse_value_array(it, end); + case parse_type::FLOAT: + return parse_value_array(it, end); + case parse_type::BOOL: + return parse_value_array(it, end); + case parse_type::ARRAY: + return parse_object_array(&parser::parse_array, '[', it, + end); + case parse_type::INLINE_TABLE: + return parse_object_array( + &parser::parse_inline_table, '{', it, end); + default: + throw_parse_exception("Unable to parse array"); + } + } + + template + std::shared_ptr parse_value_array(std::string::iterator& it, + std::string::iterator& end) + { + auto arr = make_array(); + while (it != end && *it != ']') + { + auto val = parse_value(it, end); + if (auto v = val->as()) + arr->get().push_back(val); + else + throw_parse_exception("Arrays must be homogeneous"); + skip_whitespace_and_comments(it, end); + if (*it != ',') + break; + ++it; + skip_whitespace_and_comments(it, end); + } + if (it != end) + ++it; + return arr; + } + + template + std::shared_ptr parse_object_array(Function&& fun, char delim, + std::string::iterator& it, + std::string::iterator& end) + { + auto arr = detail::make_element(); + + while (it != end && *it != ']') + { + if (*it != delim) + throw_parse_exception("Unexpected character in array"); + + arr->get().push_back(((*this).*fun)(it, end)); + skip_whitespace_and_comments(it, end); + + if (it == end || *it != ',') + break; + + ++it; + skip_whitespace_and_comments(it, end); + } + + if (it == end || *it != ']') + throw_parse_exception("Unterminated array"); + + ++it; + return arr; + } + + std::shared_ptr
parse_inline_table(std::string::iterator& it, + std::string::iterator& end) + { + auto tbl = make_table(); + do + { + ++it; + if (it == end) + throw_parse_exception("Unterminated inline table"); + + consume_whitespace(it, end); + if (it != end && *it != '}') + { + parse_key_value(it, end, tbl.get()); + consume_whitespace(it, end); + } + } while (*it == ','); + + if (it == end || *it != '}') + throw_parse_exception("Unterminated inline table"); + + ++it; + consume_whitespace(it, end); + + return tbl; + } + + void skip_whitespace_and_comments(std::string::iterator& start, + std::string::iterator& end) + { + consume_whitespace(start, end); + while (start == end || *start == '#') + { + if (!detail::getline(input_, line_)) + throw_parse_exception("Unclosed array"); + line_number_++; + start = line_.begin(); + end = line_.end(); + consume_whitespace(start, end); + } + } + + void consume_whitespace(std::string::iterator& it, + const std::string::iterator& end) + { + while (it != end && (*it == ' ' || *it == '\t')) + ++it; + } + + void consume_backwards_whitespace(std::string::iterator& back, + const std::string::iterator& front) + { + while (back != front && (*back == ' ' || *back == '\t')) + --back; + } + + void eol_or_comment(const std::string::iterator& it, + const std::string::iterator& end) + { + if (it != end && *it != '#') + throw_parse_exception("Unidentified trailing character '" + + std::string{*it} + + "'---did you forget a '#'?"); + } + + bool is_time(const std::string::iterator& it, + const std::string::iterator& end) + { + auto time_end = find_end_of_time(it, end); + auto len = std::distance(it, time_end); + + if (len < 8) + return false; + + if (it[2] != ':' || it[5] != ':') + return false; + + if (len > 8) + return it[8] == '.' && len > 9; + + return true; + } + + option date_type(const std::string::iterator& it, + const std::string::iterator& end) + { + auto date_end = find_end_of_date(it, end); + auto len = std::distance(it, date_end); + + if (len < 10) + return {}; + + if (it[4] != '-' || it[7] != '-') + return {}; + + if (len >= 19 && (it[10] == 'T' || it[10] == ' ') + && is_time(it + 11, date_end)) + { + // datetime type + auto time_end = find_end_of_time(it + 11, date_end); + if (time_end == date_end) + return {parse_type::LOCAL_DATETIME}; + else + return {parse_type::OFFSET_DATETIME}; + } + else if (len == 10) + { + // just a regular date + return {parse_type::LOCAL_DATE}; + } + + return {}; + } + + std::istream& input_; + std::string line_; + std::size_t line_number_ = 0; +}; + +/** + * Utility function to parse a file as a TOML file. Returns the root table. + * Throws a parse_exception if the file cannot be opened. + */ +inline std::shared_ptr
parse_file(const std::string& filename) +{ +#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP) + boost::nowide::ifstream file{filename.c_str()}; +#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP) + nowide::ifstream file{filename.c_str()}; +#else + std::ifstream file{filename}; +#endif + if (!file.is_open()) + throw parse_exception{filename + " could not be opened for parsing"}; + parser p{file}; + return p.parse(); +} + +template +struct value_accept; + +template <> +struct value_accept<> +{ + template + static void accept(const base&, Visitor&&, Args&&...) + { + // nothing + } +}; + +template +struct value_accept +{ + template + static void accept(const base& b, Visitor&& visitor, Args&&... args) + { + if (auto v = b.as()) + { + visitor.visit(*v, std::forward(args)...); + } + else + { + value_accept::accept(b, std::forward(visitor), + std::forward(args)...); + } + } +}; + +/** + * base implementation of accept() that calls visitor.visit() on the concrete + * class. + */ +template +void base::accept(Visitor&& visitor, Args&&... args) const +{ + if (is_value()) + { + using value_acceptor + = value_accept; + value_acceptor::accept(*this, std::forward(visitor), + std::forward(args)...); + } + else if (is_table()) + { + visitor.visit(static_cast(*this), + std::forward(args)...); + } + else if (is_array()) + { + visitor.visit(static_cast(*this), + std::forward(args)...); + } + else if (is_table_array()) + { + visitor.visit(static_cast(*this), + std::forward(args)...); + } +} + +/** + * Writer that can be passed to accept() functions of cpptoml objects and + * will output valid TOML to a stream. + */ +class toml_writer +{ + public: + /** + * Construct a toml_writer that will write to the given stream + */ + toml_writer(std::ostream& s, const std::string& indent_space = "\t") + : stream_(s), indent_(indent_space), has_naked_endline_(false) + { + // nothing + } + + public: + /** + * Output a base value of the TOML tree. + */ + template + void visit(const value& v, bool = false) + { + write(v); + } + + /** + * Output a table element of the TOML tree + */ + void visit(const table& t, bool in_array = false) + { + write_table_header(in_array); + std::vector values; + std::vector tables; + + for (const auto& i : t) + { + if (i.second->is_table() || i.second->is_table_array()) + { + tables.push_back(i.first); + } + else + { + values.push_back(i.first); + } + } + + for (unsigned int i = 0; i < values.size(); ++i) + { + path_.push_back(values[i]); + + if (i > 0) + endline(); + + write_table_item_header(*t.get(values[i])); + t.get(values[i])->accept(*this, false); + path_.pop_back(); + } + + for (unsigned int i = 0; i < tables.size(); ++i) + { + path_.push_back(tables[i]); + + if (values.size() > 0 || i > 0) + endline(); + + write_table_item_header(*t.get(tables[i])); + t.get(tables[i])->accept(*this, false); + path_.pop_back(); + } + + endline(); + } + + /** + * Output an array element of the TOML tree + */ + void visit(const array& a, bool = false) + { + write("["); + + for (unsigned int i = 0; i < a.get().size(); ++i) + { + if (i > 0) + write(", "); + + if (a.get()[i]->is_array()) + { + a.get()[i]->as_array()->accept(*this, true); + } + else + { + a.get()[i]->accept(*this, true); + } + } + + write("]"); + } + + /** + * Output a table_array element of the TOML tree + */ + void visit(const table_array& t, bool = false) + { + for (unsigned int j = 0; j < t.get().size(); ++j) + { + if (j > 0) + endline(); + + t.get()[j]->accept(*this, true); + } + + endline(); + } + + /** + * Escape a string for output. + */ + static std::string escape_string(const std::string& str) + { + std::string res; + for (auto it = str.begin(); it != str.end(); ++it) + { + if (*it == '\b') + { + res += "\\b"; + } + else if (*it == '\t') + { + res += "\\t"; + } + else if (*it == '\n') + { + res += "\\n"; + } + else if (*it == '\f') + { + res += "\\f"; + } + else if (*it == '\r') + { + res += "\\r"; + } + else if (*it == '"') + { + res += "\\\""; + } + else if (*it == '\\') + { + res += "\\\\"; + } + else if (static_cast(*it) <= UINT32_C(0x001f)) + { + res += "\\u"; + std::stringstream ss; + ss << std::hex << static_cast(*it); + res += ss.str(); + } + else + { + res += *it; + } + } + return res; + } + + protected: + /** + * Write out a string. + */ + void write(const value& v) + { + write("\""); + write(escape_string(v.get())); + write("\""); + } + + /** + * Write out a double. + */ + void write(const value& v) + { + std::stringstream ss; + ss << std::showpoint + << std::setprecision(std::numeric_limits::max_digits10) + << v.get(); + + auto double_str = ss.str(); + auto pos = double_str.find("e0"); + if (pos != std::string::npos) + double_str.replace(pos, 2, "e"); + pos = double_str.find("e-0"); + if (pos != std::string::npos) + double_str.replace(pos, 3, "e-"); + + stream_ << double_str; + has_naked_endline_ = false; + } + + /** + * Write out an integer, local_date, local_time, local_datetime, or + * offset_datetime. + */ + template + typename std::enable_if< + is_one_of::value>::type + write(const value& v) + { + write(v.get()); + } + + /** + * Write out a boolean. + */ + void write(const value& v) + { + write((v.get() ? "true" : "false")); + } + + /** + * Write out the header of a table. + */ + void write_table_header(bool in_array = false) + { + if (!path_.empty()) + { + indent(); + + write("["); + + if (in_array) + { + write("["); + } + + for (unsigned int i = 0; i < path_.size(); ++i) + { + if (i > 0) + { + write("."); + } + + if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_[i]); + } + else + { + write("\""); + write(escape_string(path_[i])); + write("\""); + } + } + + if (in_array) + { + write("]"); + } + + write("]"); + endline(); + } + } + + /** + * Write out the identifier for an item in a table. + */ + void write_table_item_header(const base& b) + { + if (!b.is_table() && !b.is_table_array()) + { + indent(); + + if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" + "fghijklmnopqrstuvwxyz0123456789" + "_-") + == std::string::npos) + { + write(path_.back()); + } + else + { + write("\""); + write(escape_string(path_.back())); + write("\""); + } + + write(" = "); + } + } + + private: + /** + * Indent the proper number of tabs given the size of + * the path. + */ + void indent() + { + for (std::size_t i = 1; i < path_.size(); ++i) + write(indent_); + } + + /** + * Write a value out to the stream. + */ + template + void write(const T& v) + { + stream_ << v; + has_naked_endline_ = false; + } + + /** + * Write an endline out to the stream + */ + void endline() + { + if (!has_naked_endline_) + { + stream_ << "\n"; + has_naked_endline_ = true; + } + } + + private: + std::ostream& stream_; + const std::string indent_; + std::vector path_; + bool has_naked_endline_; +}; + +inline std::ostream& operator<<(std::ostream& stream, const base& b) +{ + toml_writer writer{stream}; + b.accept(writer); + return stream; +} + +template +std::ostream& operator<<(std::ostream& stream, const value& v) +{ + toml_writer writer{stream}; + v.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const table_array& t) +{ + toml_writer writer{stream}; + t.accept(writer); + return stream; +} + +inline std::ostream& operator<<(std::ostream& stream, const array& a) +{ + toml_writer writer{stream}; + a.accept(writer); + return stream; +} +} // namespace cpptoml +#endif // CPPTOML_H diff --git a/third_party/nix/src/libexpr/attr-path.cc b/third_party/nix/src/libexpr/attr-path.cc new file mode 100644 index 000000000..b0f80db32 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-path.cc @@ -0,0 +1,96 @@ +#include "attr-path.hh" +#include "eval-inline.hh" +#include "util.hh" + + +namespace nix { + + +static Strings parseAttrPath(const string & s) +{ + Strings res; + string cur; + string::const_iterator i = s.begin(); + while (i != s.end()) { + if (*i == '.') { + res.push_back(cur); + cur.clear(); + } else if (*i == '"') { + ++i; + while (1) { + if (i == s.end()) + throw Error(format("missing closing quote in selection path '%1%'") % s); + if (*i == '"') break; + cur.push_back(*i++); + } + } else + cur.push_back(*i); + ++i; + } + if (!cur.empty()) res.push_back(cur); + return res; +} + + +Value * findAlongAttrPath(EvalState & state, const string & attrPath, + Bindings & autoArgs, Value & vIn) +{ + Strings tokens = parseAttrPath(attrPath); + + Error attrError = + Error(format("attribute selection path '%1%' does not match expression") % attrPath); + + Value * v = &vIn; + + for (auto & attr : tokens) { + + /* Is i an index (integer) or a normal attribute name? */ + enum { apAttr, apIndex } apType = apAttr; + unsigned int attrIndex; + if (string2Int(attr, attrIndex)) apType = apIndex; + + /* Evaluate the expression. */ + Value * vNew = state.allocValue(); + state.autoCallFunction(autoArgs, *v, *vNew); + v = vNew; + state.forceValue(*v); + + /* It should evaluate to either a set or an expression, + according to what is specified in the attrPath. */ + + if (apType == apAttr) { + + if (v->type != tAttrs) + throw TypeError( + format("the expression selected by the selection path '%1%' should be a set but is %2%") + % attrPath % showType(*v)); + + if (attr.empty()) + throw Error(format("empty attribute name in selection path '%1%'") % attrPath); + + Bindings::iterator a = v->attrs->find(state.symbols.create(attr)); + if (a == v->attrs->end()) + throw Error(format("attribute '%1%' in selection path '%2%' not found") % attr % attrPath); + v = &*a->value; + } + + else if (apType == apIndex) { + + if (!v->isList()) + throw TypeError( + format("the expression selected by the selection path '%1%' should be a list but is %2%") + % attrPath % showType(*v)); + + if (attrIndex >= v->listSize()) + throw Error(format("list index %1% in selection path '%2%' is out of range") % attrIndex % attrPath); + + v = v->listElems()[attrIndex]; + } + + } + + return v; +} + + +} diff --git a/third_party/nix/src/libexpr/attr-path.hh b/third_party/nix/src/libexpr/attr-path.hh new file mode 100644 index 000000000..46a341950 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-path.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "eval.hh" + +#include +#include + +namespace nix { + +Value * findAlongAttrPath(EvalState & state, const string & attrPath, + Bindings & autoArgs, Value & vIn); + +} diff --git a/third_party/nix/src/libexpr/attr-set.cc b/third_party/nix/src/libexpr/attr-set.cc new file mode 100644 index 000000000..0785897d2 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-set.cc @@ -0,0 +1,52 @@ +#include "attr-set.hh" +#include "eval-inline.hh" + +#include + + +namespace nix { + + +/* Allocate a new array of attributes for an attribute set with a specific + capacity. The space is implicitly reserved after the Bindings + structure. */ +Bindings * EvalState::allocBindings(size_t capacity) +{ + if (capacity > std::numeric_limits::max()) + throw Error("attribute set of size %d is too big", capacity); + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); +} + + +void EvalState::mkAttrs(Value & v, size_t capacity) +{ + if (capacity == 0) { + v = vEmptySet; + return; + } + clearValue(v); + v.type = tAttrs; + v.attrs = allocBindings(capacity); + nrAttrsets++; + nrAttrsInAttrsets += capacity; +} + + +/* Create a new attribute named 'name' on an existing attribute set stored + in 'vAttrs' and return the newly allocated Value which is associated with + this attribute. */ +Value * EvalState::allocAttr(Value & vAttrs, const Symbol & name) +{ + Value * v = allocValue(); + vAttrs.attrs->push_back(Attr(name, v)); + return v; +} + + +void Bindings::sort() +{ + std::sort(begin(), end()); +} + + +} diff --git a/third_party/nix/src/libexpr/attr-set.hh b/third_party/nix/src/libexpr/attr-set.hh new file mode 100644 index 000000000..3119a1848 --- /dev/null +++ b/third_party/nix/src/libexpr/attr-set.hh @@ -0,0 +1,95 @@ +#pragma once + +#include "nixexpr.hh" +#include "symbol-table.hh" + +#include + +namespace nix { + + +class EvalState; +struct Value; + +/* Map one attribute name to its value. */ +struct Attr +{ + Symbol name; + Value * value; + Pos * pos; + Attr(Symbol name, Value * value, Pos * pos = &noPos) + : name(name), value(value), pos(pos) { }; + Attr() : pos(&noPos) { }; + bool operator < (const Attr & a) const + { + return name < a.name; + } +}; + +/* Bindings contains all the attributes of an attribute set. It is defined + by its size and its capacity, the capacity being the number of Attr + elements allocated after this structure, while the size corresponds to + the number of elements already inserted in this structure. */ +class Bindings +{ +public: + typedef uint32_t size_t; + +private: + size_t size_, capacity_; + Attr attrs[0]; + + Bindings(size_t capacity) : size_(0), capacity_(capacity) { } + Bindings(const Bindings & bindings) = delete; + +public: + size_t size() const { return size_; } + + bool empty() const { return !size_; } + + typedef Attr * iterator; + + void push_back(const Attr & attr) + { + assert(size_ < capacity_); + attrs[size_++] = attr; + } + + iterator find(const Symbol & name) + { + Attr key(name, 0); + iterator i = std::lower_bound(begin(), end(), key); + if (i != end() && i->name == name) return i; + return end(); + } + + iterator begin() { return &attrs[0]; } + iterator end() { return &attrs[size_]; } + + Attr & operator[](size_t pos) + { + return attrs[pos]; + } + + void sort(); + + size_t capacity() { return capacity_; } + + /* Returns the attributes in lexicographically sorted order. */ + std::vector lexicographicOrder() const + { + std::vector res; + res.reserve(size_); + for (size_t n = 0; n < size_; n++) + res.emplace_back(&attrs[n]); + std::sort(res.begin(), res.end(), [](const Attr * a, const Attr * b) { + return (const string &) a->name < (const string &) b->name; + }); + return res; + } + + friend class EvalState; +}; + + +} diff --git a/third_party/nix/src/libexpr/common-eval-args.cc b/third_party/nix/src/libexpr/common-eval-args.cc new file mode 100644 index 000000000..13950ab8d --- /dev/null +++ b/third_party/nix/src/libexpr/common-eval-args.cc @@ -0,0 +1,59 @@ +#include "common-eval-args.hh" +#include "shared.hh" +#include "download.hh" +#include "util.hh" +#include "eval.hh" + +namespace nix { + +MixEvalArgs::MixEvalArgs() +{ + mkFlag() + .longName("arg") + .description("argument to be passed to Nix functions") + .labels({"name", "expr"}) + .handler([&](std::vector ss) { autoArgs[ss[0]] = 'E' + ss[1]; }); + + mkFlag() + .longName("argstr") + .description("string-valued argument to be passed to Nix functions") + .labels({"name", "string"}) + .handler([&](std::vector ss) { autoArgs[ss[0]] = 'S' + ss[1]; }); + + mkFlag() + .shortName('I') + .longName("include") + .description("add a path to the list of locations used to look up <...> file names") + .label("path") + .handler([&](std::string s) { searchPath.push_back(s); }); +} + +Bindings * MixEvalArgs::getAutoArgs(EvalState & state) +{ + Bindings * res = state.allocBindings(autoArgs.size()); + for (auto & i : autoArgs) { + Value * v = state.allocValue(); + if (i.second[0] == 'E') + state.mkThunk_(*v, state.parseExprFromString(string(i.second, 1), absPath("."))); + else + mkString(*v, string(i.second, 1)); + res->push_back(Attr(state.symbols.create(i.first), v)); + } + res->sort(); + return res; +} + +Path lookupFileArg(EvalState & state, string s) +{ + if (isUri(s)) { + CachedDownloadRequest request(s); + request.unpack = true; + return getDownloader()->downloadCached(state.store, request).path; + } else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') { + Path p = s.substr(1, s.size() - 2); + return state.findFile(p); + } else + return absPath(s); +} + +} diff --git a/third_party/nix/src/libexpr/common-eval-args.hh b/third_party/nix/src/libexpr/common-eval-args.hh new file mode 100644 index 000000000..be7fda783 --- /dev/null +++ b/third_party/nix/src/libexpr/common-eval-args.hh @@ -0,0 +1,26 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +class Store; +class EvalState; +class Bindings; + +struct MixEvalArgs : virtual Args +{ + MixEvalArgs(); + + Bindings * getAutoArgs(EvalState & state); + + Strings searchPath; + +private: + + std::map autoArgs; +}; + +Path lookupFileArg(EvalState & state, string s); + +} diff --git a/third_party/nix/src/libexpr/eval-inline.hh b/third_party/nix/src/libexpr/eval-inline.hh new file mode 100644 index 000000000..c27116e3b --- /dev/null +++ b/third_party/nix/src/libexpr/eval-inline.hh @@ -0,0 +1,95 @@ +#pragma once + +#include "eval.hh" + +#define LocalNoInline(f) static f __attribute__((noinline)); f +#define LocalNoInlineNoReturn(f) static f __attribute__((noinline, noreturn)); f + +namespace nix { + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const Pos & pos)) +{ + throw EvalError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v)) +{ + throw TypeError(format(s) % showType(v)); +} + + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v, const Pos & pos)) +{ + throw TypeError(format(s) % showType(v) % pos); +} + + +void EvalState::forceValue(Value & v, const Pos & pos) +{ + if (v.type == tThunk) { + Env * env = v.thunk.env; + Expr * expr = v.thunk.expr; + try { + v.type = tBlackhole; + //checkInterrupt(); + expr->eval(*this, *env, v); + } catch (...) { + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = expr; + throw; + } + } + else if (v.type == tApp) + callFunction(*v.app.left, *v.app.right, v, noPos); + else if (v.type == tBlackhole) + throwEvalError("infinite recursion encountered, at %1%", pos); +} + + +inline void EvalState::forceAttrs(Value & v) +{ + forceValue(v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected", v); +} + + +inline void EvalState::forceAttrs(Value & v, const Pos & pos) +{ + forceValue(v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected, at %2%", v, pos); +} + + +inline void EvalState::forceList(Value & v) +{ + forceValue(v); + if (!v.isList()) + throwTypeError("value is %1% while a list was expected", v); +} + + +inline void EvalState::forceList(Value & v, const Pos & pos) +{ + forceValue(v); + if (!v.isList()) + throwTypeError("value is %1% while a list was expected, at %2%", v, pos); +} + +/* Note: Various places expect the allocated memory to be zeroed. */ +inline void * allocBytes(size_t n) +{ + void * p; +#if HAVE_BOEHMGC + p = GC_MALLOC(n); +#else + p = calloc(n, 1); +#endif + if (!p) throw std::bad_alloc(); + return p; +} + + +} diff --git a/third_party/nix/src/libexpr/eval.cc b/third_party/nix/src/libexpr/eval.cc new file mode 100644 index 000000000..3426afb6c --- /dev/null +++ b/third_party/nix/src/libexpr/eval.cc @@ -0,0 +1,1991 @@ +#include "eval.hh" +#include "hash.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "globals.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "json.hh" +#include "function-trace.hh" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#if HAVE_BOEHMGC + +#include +#include + +#endif + +namespace nix { + + +static char * dupString(const char * s) +{ + char * t; +#if HAVE_BOEHMGC + t = GC_STRDUP(s); +#else + t = strdup(s); +#endif + if (!t) throw std::bad_alloc(); + return t; +} + + +static void printValue(std::ostream & str, std::set & active, const Value & v) +{ + checkInterrupt(); + + if (active.find(&v) != active.end()) { + str << ""; + return; + } + active.insert(&v); + + switch (v.type) { + case tInt: + str << v.integer; + break; + case tBool: + str << (v.boolean ? "true" : "false"); + break; + case tString: + str << "\""; + for (const char * i = v.string.s; *i; i++) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; + break; + case tPath: + str << v.path; // !!! escaping? + break; + case tNull: + str << "null"; + break; + case tAttrs: { + str << "{ "; + for (auto & i : v.attrs->lexicographicOrder()) { + str << i->name << " = "; + printValue(str, active, *i->value); + str << "; "; + } + str << "}"; + break; + } + case tList1: + case tList2: + case tListN: + str << "[ "; + for (unsigned int n = 0; n < v.listSize(); ++n) { + printValue(str, active, *v.listElems()[n]); + str << " "; + } + str << "]"; + break; + case tThunk: + case tApp: + str << ""; + break; + case tLambda: + str << ""; + break; + case tPrimOp: + str << ""; + break; + case tPrimOpApp: + str << ""; + break; + case tExternal: + str << *v.external; + break; + case tFloat: + str << v.fpoint; + break; + default: + throw Error("invalid value"); + } + + active.erase(&v); +} + + +std::ostream & operator << (std::ostream & str, const Value & v) +{ + std::set active; + printValue(str, active, v); + return str; +} + + +const Value *getPrimOp(const Value &v) { + const Value * primOp = &v; + while (primOp->type == tPrimOpApp) { + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + return primOp; +} + + +string showType(const Value & v) +{ + switch (v.type) { + case tInt: return "an integer"; + case tBool: return "a boolean"; + case tString: return v.string.context ? "a string with context" : "a string"; + case tPath: return "a path"; + case tNull: return "null"; + case tAttrs: return "a set"; + case tList1: case tList2: case tListN: return "a list"; + case tThunk: return "a thunk"; + case tApp: return "a function application"; + case tLambda: return "a function"; + case tBlackhole: return "a black hole"; + case tPrimOp: + return fmt("the built-in function '%s'", string(v.primOp->name)); + case tPrimOpApp: + return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); + case tExternal: return v.external->showType(); + case tFloat: return "a float"; + } + abort(); +} + + +#if HAVE_BOEHMGC +/* Called when the Boehm GC runs out of memory. */ +static void * oomHandler(size_t requested) +{ + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); +} +#endif + + +static Symbol getName(const AttrName & name, EvalState & state, Env & env) +{ + if (name.symbol.set()) { + return name.symbol; + } else { + Value nameValue; + name.expr->eval(state, env, nameValue); + state.forceStringNoCtx(nameValue); + return state.symbols.create(nameValue.string.s); + } +} + + +static bool gcInitialised = false; + +void initGC() +{ + if (gcInitialised) return; + +#if HAVE_BOEHMGC + /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ + GC_set_all_interior_pointers(0); + + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + + GC_INIT(); + + GC_set_oom_fn(oomHandler); + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most cases + we don't need to garbage collect at all. (Collection has a + fairly significant overhead.) The heap size can be overridden + through libgc's GC_INITIAL_HEAP_SIZE environment variable. We + should probably also provide a nix.conf setting for this. Note + that GC_expand_hp() causes a lot of virtual, but not physical + (resident) memory to be allocated. This might be a problem on + systems that don't overcommit. */ + if (!getenv("GC_INITIAL_HEAP_SIZE")) { + size_t size = 32 * 1024 * 1024; +#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + size_t maxSize = 384 * 1024 * 1024; + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) + size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) size = maxSize; +#endif + debug(format("setting initial heap size to %1% bytes") % size); + GC_expand_hp(size); + } + +#endif + + gcInitialised = true; +} + + +/* Very hacky way to parse $NIX_PATH, which is colon-separated, but + can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ +static Strings parseNixPath(const string & s) +{ + Strings res; + + auto p = s.begin(); + + while (p != s.end()) { + auto start = p; + auto start2 = p; + + while (p != s.end() && *p != ':') { + if (*p == '=') start2 = p + 1; + ++p; + } + + if (p == s.end()) { + if (p != start) res.push_back(std::string(start, p)); + break; + } + + if (*p == ':') { + if (isUri(std::string(start2, s.end()))) { + ++p; + while (p != s.end() && *p != ':') ++p; + } + res.push_back(std::string(start, p)); + if (p == s.end()) break; + } + + ++p; + } + + return res; +} + + +EvalState::EvalState(const Strings & _searchPath, ref store) + : sWith(symbols.create("")) + , sOutPath(symbols.create("outPath")) + , sDrvPath(symbols.create("drvPath")) + , sType(symbols.create("type")) + , sMeta(symbols.create("meta")) + , sName(symbols.create("name")) + , sValue(symbols.create("value")) + , sSystem(symbols.create("system")) + , sOverrides(symbols.create("__overrides")) + , sOutputs(symbols.create("outputs")) + , sOutputName(symbols.create("outputName")) + , sIgnoreNulls(symbols.create("__ignoreNulls")) + , sFile(symbols.create("file")) + , sLine(symbols.create("line")) + , sColumn(symbols.create("column")) + , sFunctor(symbols.create("__functor")) + , sToString(symbols.create("__toString")) + , sRight(symbols.create("right")) + , sWrong(symbols.create("wrong")) + , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sBuilder(symbols.create("builder")) + , sArgs(symbols.create("args")) + , sOutputHash(symbols.create("outputHash")) + , sOutputHashAlgo(symbols.create("outputHashAlgo")) + , sOutputHashMode(symbols.create("outputHashMode")) + , repair(NoRepair) + , store(store) + , baseEnv(allocEnv(128)) + , staticBaseEnv(false, 0) +{ + countCalls = getEnv("NIX_COUNT_CALLS", "0") != "0"; + + assert(gcInitialised); + + static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); + + /* Initialise the Nix expression search path. */ + if (!evalSettings.pureEval) { + Strings paths = parseNixPath(getEnv("NIX_PATH", "")); + for (auto & i : _searchPath) addToSearchPath(i); + for (auto & i : paths) addToSearchPath(i); + } + addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); + + if (evalSettings.restrictEval || evalSettings.pureEval) { + allowedPaths = PathSet(); + + for (auto & i : searchPath) { + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + + auto path = r.second; + + if (store->isInStore(r.second)) { + PathSet closure; + store->computeFSClosure(store->toStorePath(r.second), closure); + for (auto & path : closure) + allowedPaths->insert(path); + } else + allowedPaths->insert(r.second); + } + } + + clearValue(vEmptySet); + vEmptySet.type = tAttrs; + vEmptySet.attrs = allocBindings(0); + + createBaseEnv(); +} + + +EvalState::~EvalState() +{ +} + + +Path EvalState::checkSourcePath(const Path & path_) +{ + if (!allowedPaths) return path_; + + auto i = resolvedPaths.find(path_); + if (i != resolvedPaths.end()) + return i->second; + + bool found = false; + + /* First canonicalize the path without symlinks, so we make sure an + * attacker can't append ../../... to a path that would be in allowedPaths + * and thus leak symlink targets. + */ + Path abspath = canonPath(path_); + + for (auto & i : *allowedPaths) { + if (isDirOrInDir(abspath, i)) { + found = true; + break; + } + } + + if (!found) + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", abspath); + + /* Resolve symlinks. */ + debug(format("checking access to '%s'") % abspath); + Path path = canonPath(abspath, true); + + for (auto & i : *allowedPaths) { + if (isDirOrInDir(path, i)) { + resolvedPaths[path_] = path; + return path; + } + } + + throw RestrictedPathError("access to path '%1%' is forbidden in restricted mode", path); +} + + +void EvalState::checkURI(const std::string & uri) +{ + if (!evalSettings.restrictEval) return; + + /* 'uri' should be equal to a prefix, or in a subdirectory of a + prefix. Thus, the prefix https://github.co does not permit + access to https://github.com. Note: this allows 'http://' and + 'https://' as prefixes for any http/https URI. */ + for (auto & prefix : evalSettings.allowedUris.get()) + if (uri == prefix || + (uri.size() > prefix.size() + && prefix.size() > 0 + && hasPrefix(uri, prefix) + && (prefix[prefix.size() - 1] == '/' || uri[prefix.size()] == '/'))) + return; + + /* If the URI is a path, then check it against allowedPaths as + well. */ + if (hasPrefix(uri, "/")) { + checkSourcePath(uri); + return; + } + + if (hasPrefix(uri, "file://")) { + checkSourcePath(std::string(uri, 7)); + return; + } + + throw RestrictedPathError("access to URI '%s' is forbidden in restricted mode", uri); +} + + +Path EvalState::toRealPath(const Path & path, const PathSet & context) +{ + // FIXME: check whether 'path' is in 'context'. + return + !context.empty() && store->isInStore(path) + ? store->toRealPath(path) + : path; +}; + + +Value * EvalState::addConstant(const string & name, Value & v) +{ + Value * v2 = allocValue(); + *v2 = v; + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v2; + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + baseEnv.values[0]->attrs->push_back(Attr(symbols.create(name2), v2)); + return v2; +} + + +Value * EvalState::addPrimOp(const string & name, + size_t arity, PrimOpFun primOp) +{ + if (arity == 0) { + Value v; + primOp(*this, noPos, nullptr, v); + return addConstant(name, v); + } + Value * v = allocValue(); + string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + v->type = tPrimOp; + v->primOp = new PrimOp(primOp, arity, sym); + staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; + baseEnv.values[baseEnvDispl++] = v; + baseEnv.values[0]->attrs->push_back(Attr(sym, v)); + return v; +} + + +Value & EvalState::getBuiltin(const string & name) +{ + return *baseEnv.values[0]->attrs->find(symbols.create(name))->value; +} + + +/* Every "format" object (even temporary) takes up a few hundred bytes + of stack space, which is a real killer in the recursive + evaluator. So here are some helper functions for throwing + exceptions. */ + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2)) +{ + throw EvalError(format(s) % s2); +} + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const Pos & pos)) +{ + throw EvalError(format(s) % s2 % pos); +} + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3)) +{ + throw EvalError(format(s) % s2 % s3); +} + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, const string & s3, const Pos & pos)) +{ + throw EvalError(format(s) % s2 % s3 % pos); +} + +LocalNoInlineNoReturn(void throwEvalError(const char * s, const Symbol & sym, const Pos & p1, const Pos & p2)) +{ + throw EvalError(format(s) % sym % p1 % p2); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const Pos & pos)) +{ + throw TypeError(format(s) % pos); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) +{ + throw TypeError(format(s) % s1); +} + +LocalNoInlineNoReturn(void throwTypeError(const char * s, const ExprLambda & fun, const Symbol & s2, const Pos & pos)) +{ + throw TypeError(format(s) % fun.showNamePos() % s2 % pos); +} + +LocalNoInlineNoReturn(void throwAssertionError(const char * s, const string & s1, const Pos & pos)) +{ + throw AssertionError(format(s) % s1 % pos); +} + +LocalNoInlineNoReturn(void throwUndefinedVarError(const char * s, const string & s1, const Pos & pos)) +{ + throw UndefinedVarError(format(s) % s1 % pos); +} + +LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2)) +{ + e.addPrefix(format(s) % s2); +} + +LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos)) +{ + e.addPrefix(format(s) % fun.showNamePos() % pos); +} + +LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos)) +{ + e.addPrefix(format(s) % s2 % pos); +} + + +void mkString(Value & v, const char * s) +{ + mkStringNoCopy(v, dupString(s)); +} + + +Value & mkString(Value & v, const string & s, const PathSet & context) +{ + mkString(v, s.c_str()); + if (!context.empty()) { + size_t n = 0; + v.string.context = (const char * *) + allocBytes((context.size() + 1) * sizeof(char *)); + for (auto & i : context) + v.string.context[n++] = dupString(i.c_str()); + v.string.context[n] = 0; + } + return v; +} + + +void mkPath(Value & v, const char * s) +{ + mkPathNoCopy(v, dupString(s)); +} + + +inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) +{ + for (size_t l = var.level; l; --l, env = env->up) ; + + if (!var.fromWith) return env->values[var.displ]; + + while (1) { + if (env->type == Env::HasWithExpr) { + if (noEval) return 0; + Value * v = allocValue(); + evalAttrs(*env->up, (Expr *) env->values[0], *v); + env->values[0] = v; + env->type = Env::HasWithAttrs; + } + Bindings::iterator j = env->values[0]->attrs->find(var.name); + if (j != env->values[0]->attrs->end()) { + if (countCalls && j->pos) attrSelects[*j->pos]++; + return j->value; + } + if (!env->prevWith) + throwUndefinedVarError("undefined variable '%1%' at %2%", var.name, var.pos); + for (size_t l = env->prevWith; l; --l, env = env->up) ; + } +} + + +std::atomic nrValuesFreed{0}; + +void finalizeValue(void * obj, void * data) +{ + nrValuesFreed++; +} + +Value * EvalState::allocValue() +{ + nrValues++; + auto v = (Value *) allocBytes(sizeof(Value)); + //GC_register_finalizer_no_order(v, finalizeValue, nullptr, nullptr, nullptr); + return v; +} + + +Env & EvalState::allocEnv(size_t size) +{ + if (size > std::numeric_limits::max()) + throw Error("environment size %d is too big", size); + + nrEnvs++; + nrValuesInEnvs += size; + Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *)); + env->size = (decltype(Env::size)) size; + env->type = Env::Plain; + + /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */ + + return *env; +} + + +void EvalState::mkList(Value & v, size_t size) +{ + clearValue(v); + if (size == 1) + v.type = tList1; + else if (size == 2) + v.type = tList2; + else { + v.type = tListN; + v.bigList.size = size; + v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; + } + nrListElems += size; +} + + +unsigned long nrThunks = 0; + +static inline void mkThunk(Value & v, Env & env, Expr * expr) +{ + v.type = tThunk; + v.thunk.env = &env; + v.thunk.expr = expr; + nrThunks++; +} + + +void EvalState::mkThunk_(Value & v, Expr * expr) +{ + mkThunk(v, baseEnv, expr); +} + + +void EvalState::mkPos(Value & v, Pos * pos) +{ + if (pos && pos->file.set()) { + mkAttrs(v, 3); + mkString(*allocAttr(v, sFile), pos->file); + mkInt(*allocAttr(v, sLine), pos->line); + mkInt(*allocAttr(v, sColumn), pos->column); + v.attrs->sort(); + } else + mkNull(v); +} + + +/* Create a thunk for the delayed computation of the given expression + in the given environment. But if the expression is a variable, + then look it up right away. This significantly reduces the number + of thunks allocated. */ +Value * Expr::maybeThunk(EvalState & state, Env & env) +{ + Value * v = state.allocValue(); + mkThunk(*v, env, this); + return v; +} + + +unsigned long nrAvoided = 0; + +Value * ExprVar::maybeThunk(EvalState & state, Env & env) +{ + Value * v = state.lookupVar(&env, *this, true); + /* The value might not be initialised in the environment yet. + In that case, ignore it. */ + if (v) { nrAvoided++; return v; } + return Expr::maybeThunk(state, env); +} + + +Value * ExprString::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + +Value * ExprInt::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + +Value * ExprFloat::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + +Value * ExprPath::maybeThunk(EvalState & state, Env & env) +{ + nrAvoided++; + return &v; +} + + +void EvalState::evalFile(const Path & path_, Value & v) +{ + auto path = checkSourcePath(path_); + + FileEvalCache::iterator i; + if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { + v = i->second; + return; + } + + Path path2 = resolveExprPath(path); + if ((i = fileEvalCache.find(path2)) != fileEvalCache.end()) { + v = i->second; + return; + } + + printTalkative("evaluating file '%1%'", path2); + Expr * e = nullptr; + + auto j = fileParseCache.find(path2); + if (j != fileParseCache.end()) + e = j->second; + + if (!e) + e = parseExprFromFile(checkSourcePath(path2)); + + fileParseCache[path2] = e; + + try { + eval(e, v); + } catch (Error & e) { + addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); + throw; + } + + fileEvalCache[path2] = v; + if (path != path2) fileEvalCache[path] = v; +} + + +void EvalState::resetFileCache() +{ + fileEvalCache.clear(); + fileParseCache.clear(); +} + + +void EvalState::eval(Expr * e, Value & v) +{ + e->eval(*this, baseEnv, v); +} + + +inline bool EvalState::evalBool(Env & env, Expr * e) +{ + Value v; + e->eval(*this, env, v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected", v); + return v.boolean; +} + + +inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) +{ + Value v; + e->eval(*this, env, v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + return v.boolean; +} + + +inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) +{ + e->eval(*this, env, v); + if (v.type != tAttrs) + throwTypeError("value is %1% while a set was expected", v); +} + + +void Expr::eval(EvalState & state, Env & env, Value & v) +{ + abort(); +} + + +void ExprInt::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + + +void ExprFloat::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + +void ExprString::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + + +void ExprPath::eval(EvalState & state, Env & env, Value & v) +{ + v = this->v; +} + + +void ExprAttrs::eval(EvalState & state, Env & env, Value & v) +{ + state.mkAttrs(v, attrs.size() + dynamicAttrs.size()); + Env *dynamicEnv = &env; + + if (recursive) { + /* Create a new environment that contains the attributes in + this `rec'. */ + Env & env2(state.allocEnv(attrs.size())); + env2.up = &env; + dynamicEnv = &env2; + + AttrDefs::iterator overrides = attrs.find(state.sOverrides); + bool hasOverrides = overrides != attrs.end(); + + /* The recursive attributes are evaluated in the new + environment, while the inherited attributes are evaluated + in the original environment. */ + size_t displ = 0; + for (auto & i : attrs) { + Value * vAttr; + if (hasOverrides && !i.second.inherited) { + vAttr = state.allocValue(); + mkThunk(*vAttr, env2, i.second.e); + } else + vAttr = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + env2.values[displ++] = vAttr; + v.attrs->push_back(Attr(i.first, vAttr, &i.second.pos)); + } + + /* If the rec contains an attribute called `__overrides', then + evaluate it, and add the attributes in that set to the rec. + This allows overriding of recursive attributes, which is + otherwise not possible. (You can use the // operator to + replace an attribute, but other attributes in the rec will + still reference the original value, because that value has + been substituted into the bodies of the other attributes. + Hence we need __overrides.) */ + if (hasOverrides) { + Value * vOverrides = (*v.attrs)[overrides->second.displ].value; + state.forceAttrs(*vOverrides); + Bindings * newBnds = state.allocBindings(v.attrs->capacity() + vOverrides->attrs->size()); + for (auto & i : *v.attrs) + newBnds->push_back(i); + for (auto & i : *vOverrides->attrs) { + AttrDefs::iterator j = attrs.find(i.name); + if (j != attrs.end()) { + (*newBnds)[j->second.displ] = i; + env2.values[j->second.displ] = i.value; + } else + newBnds->push_back(i); + } + newBnds->sort(); + v.attrs = newBnds; + } + } + + else + for (auto & i : attrs) + v.attrs->push_back(Attr(i.first, i.second.e->maybeThunk(state, env), &i.second.pos)); + + /* Dynamic attrs apply *after* rec and __overrides. */ + for (auto & i : dynamicAttrs) { + Value nameVal; + i.nameExpr->eval(state, *dynamicEnv, nameVal); + state.forceValue(nameVal, i.pos); + if (nameVal.type == tNull) + continue; + state.forceStringNoCtx(nameVal); + Symbol nameSym = state.symbols.create(nameVal.string.s); + Bindings::iterator j = v.attrs->find(nameSym); + if (j != v.attrs->end()) + throwEvalError("dynamic attribute '%1%' at %2% already defined at %3%", nameSym, i.pos, *j->pos); + + i.valueExpr->setName(nameSym); + /* Keep sorted order so find can catch duplicates */ + v.attrs->push_back(Attr(nameSym, i.valueExpr->maybeThunk(state, *dynamicEnv), &i.pos)); + v.attrs->sort(); // FIXME: inefficient + } +} + + +void ExprLet::eval(EvalState & state, Env & env, Value & v) +{ + /* Create a new environment that contains the attributes in this + `let'. */ + Env & env2(state.allocEnv(attrs->attrs.size())); + env2.up = &env; + + /* The recursive attributes are evaluated in the new environment, + while the inherited attributes are evaluated in the original + environment. */ + size_t displ = 0; + for (auto & i : attrs->attrs) + env2.values[displ++] = i.second.e->maybeThunk(state, i.second.inherited ? env : env2); + + body->eval(state, env2, v); +} + + +void ExprList::eval(EvalState & state, Env & env, Value & v) +{ + state.mkList(v, elems.size()); + for (size_t n = 0; n < elems.size(); ++n) + v.listElems()[n] = elems[n]->maybeThunk(state, env); +} + + +void ExprVar::eval(EvalState & state, Env & env, Value & v) +{ + Value * v2 = state.lookupVar(&env, *this, false); + state.forceValue(*v2, pos); + v = *v2; +} + + +static string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) +{ + std::ostringstream out; + bool first = true; + for (auto & i : attrPath) { + if (!first) out << '.'; else first = false; + try { + out << getName(i, state, env); + } catch (Error & e) { + assert(!i.symbol.set()); + out << "\"${" << *i.expr << "}\""; + } + } + return out.str(); +} + + +unsigned long nrLookups = 0; + +void ExprSelect::eval(EvalState & state, Env & env, Value & v) +{ + Value vTmp; + Pos * pos2 = 0; + Value * vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + try { + + for (auto & i : attrPath) { + nrLookups++; + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (def) { + state.forceValue(*vAttrs, pos); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + { + def->eval(state, env, v); + return; + } + } else { + state.forceAttrs(*vAttrs, pos); + if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + throwEvalError("attribute '%1%' missing, at %2%", name, pos); + } + vAttrs = j->value; + pos2 = j->pos; + if (state.countCalls && pos2) state.attrSelects[*pos2]++; + } + + state.forceValue(*vAttrs, ( pos2 != NULL ? *pos2 : this->pos ) ); + + } catch (Error & e) { + if (pos2 && pos2->file != state.sDerivationNix) + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", + showAttrPath(state, env, attrPath), *pos2); + throw; + } + + v = *vAttrs; +} + + +void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) +{ + Value vTmp; + Value * vAttrs = &vTmp; + + e->eval(state, env, vTmp); + + for (auto & i : attrPath) { + state.forceValue(*vAttrs); + Bindings::iterator j; + Symbol name = getName(i, state, env); + if (vAttrs->type != tAttrs || + (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) + { + mkBool(v, false); + return; + } else { + vAttrs = j->value; + } + } + + mkBool(v, true); +} + + +void ExprLambda::eval(EvalState & state, Env & env, Value & v) +{ + v.type = tLambda; + v.lambda.env = &env; + v.lambda.fun = this; +} + + +void ExprApp::eval(EvalState & state, Env & env, Value & v) +{ + /* FIXME: vFun prevents GCC from doing tail call optimisation. */ + Value vFun; + e1->eval(state, env, vFun); + state.callFunction(vFun, *(e2->maybeThunk(state, env)), v, pos); +} + + +void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) +{ + /* Figure out the number of arguments still needed. */ + size_t argsDone = 0; + Value * primOp = &fun; + while (primOp->type == tPrimOpApp) { + argsDone++; + primOp = primOp->primOpApp.left; + } + assert(primOp->type == tPrimOp); + auto arity = primOp->primOp->arity; + auto argsLeft = arity - argsDone; + + if (argsLeft == 1) { + /* We have all the arguments, so call the primop. */ + + /* Put all the arguments in an array. */ + Value * vArgs[arity]; + auto n = arity - 1; + vArgs[n--] = &arg; + for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + vArgs[n--] = arg->primOpApp.right; + + /* And call the primop. */ + nrPrimOpCalls++; + if (countCalls) primOpCalls[primOp->primOp->name]++; + primOp->primOp->fun(*this, pos, vArgs, v); + } else { + Value * fun2 = allocValue(); + *fun2 = fun; + v.type = tPrimOpApp; + v.primOpApp.left = fun2; + v.primOpApp.right = &arg; + } +} + +void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & pos) +{ + auto trace = evalSettings.traceFunctionCalls ? std::make_unique(pos) : nullptr; + + forceValue(fun, pos); + + if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + callPrimOp(fun, arg, v, pos); + return; + } + + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + /* fun may be allocated on the stack of the calling function, + * but for functors we may keep a reference, so heap-allocate + * a copy and use that instead. + */ + auto & fun2 = *allocValue(); + fun2 = fun; + /* !!! Should we use the attr pos here? */ + Value v2; + callFunction(*found->value, fun2, v2, pos); + return callFunction(v2, arg, v, pos); + } + } + + if (fun.type != tLambda) + throwTypeError("attempt to call something which is not a function but %1%, at %2%", fun, pos); + + ExprLambda & lambda(*fun.lambda.fun); + + auto size = + (lambda.arg.empty() ? 0 : 1) + + (lambda.matchAttrs ? lambda.formals->formals.size() : 0); + Env & env2(allocEnv(size)); + env2.up = fun.lambda.env; + + size_t displ = 0; + + if (!lambda.matchAttrs) + env2.values[displ++] = &arg; + + else { + forceAttrs(arg, pos); + + if (!lambda.arg.empty()) + env2.values[displ++] = &arg; + + /* For each formal argument, get the actual argument. If + there is no matching actual argument but the formal + argument has a default, use the default. */ + size_t attrsUsed = 0; + for (auto & i : lambda.formals->formals) { + Bindings::iterator j = arg.attrs->find(i.name); + if (j == arg.attrs->end()) { + if (!i.def) throwTypeError("%1% called without required argument '%2%', at %3%", + lambda, i.name, pos); + env2.values[displ++] = i.def->maybeThunk(*this, env2); + } else { + attrsUsed++; + env2.values[displ++] = j->value; + } + } + + /* Check that each actual argument is listed as a formal + argument (unless the attribute match specifies a `...'). */ + if (!lambda.formals->ellipsis && attrsUsed != arg.attrs->size()) { + /* Nope, so show the first unexpected argument to the + user. */ + for (auto & i : *arg.attrs) + if (lambda.formals->argNames.find(i.name) == lambda.formals->argNames.end()) + throwTypeError("%1% called with unexpected argument '%2%', at %3%", lambda, i.name, pos); + abort(); // can't happen + } + } + + nrFunctionCalls++; + if (countCalls) incrFunctionCall(&lambda); + + /* Evaluate the body. This is conditional on showTrace, because + catching exceptions makes this function not tail-recursive. */ + if (settings.showTrace) + try { + lambda.body->eval(*this, env2, v); + } catch (Error & e) { + addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos); + throw; + } + else + fun.lambda.fun->body->eval(*this, env2, v); +} + + +// Lifted out of callFunction() because it creates a temporary that +// prevents tail-call optimisation. +void EvalState::incrFunctionCall(ExprLambda * fun) +{ + functionCalls[fun]++; +} + + +void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) +{ + forceValue(fun); + + if (fun.type == tAttrs) { + auto found = fun.attrs->find(sFunctor); + if (found != fun.attrs->end()) { + Value * v = allocValue(); + callFunction(*found->value, fun, *v, noPos); + forceValue(*v); + return autoCallFunction(args, *v, res); + } + } + + if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + res = fun; + return; + } + + Value * actualArgs = allocValue(); + mkAttrs(*actualArgs, fun.lambda.fun->formals->formals.size()); + + for (auto & i : fun.lambda.fun->formals->formals) { + Bindings::iterator j = args.find(i.name); + if (j != args.end()) + actualArgs->attrs->push_back(*j); + else if (!i.def) + throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + } + + actualArgs->attrs->sort(); + + callFunction(fun, *actualArgs, res, noPos); +} + + +void ExprWith::eval(EvalState & state, Env & env, Value & v) +{ + Env & env2(state.allocEnv(1)); + env2.up = &env; + env2.prevWith = prevWith; + env2.type = Env::HasWithExpr; + env2.values[0] = (Value *) attrs; + + body->eval(state, env2, v); +} + + +void ExprIf::eval(EvalState & state, Env & env, Value & v) +{ + (state.evalBool(env, cond) ? then : else_)->eval(state, env, v); +} + + +void ExprAssert::eval(EvalState & state, Env & env, Value & v) +{ + if (!state.evalBool(env, cond, pos)) { + std::ostringstream out; + cond->show(out); + throwAssertionError("assertion %1% failed at %2%", out.str(), pos); + } + body->eval(state, env, v); +} + + +void ExprOpNot::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, !state.evalBool(env, e)); +} + + +void ExprOpEq::eval(EvalState & state, Env & env, Value & v) +{ + Value v1; e1->eval(state, env, v1); + Value v2; e2->eval(state, env, v2); + mkBool(v, state.eqValues(v1, v2)); +} + + +void ExprOpNEq::eval(EvalState & state, Env & env, Value & v) +{ + Value v1; e1->eval(state, env, v1); + Value v2; e2->eval(state, env, v2); + mkBool(v, !state.eqValues(v1, v2)); +} + + +void ExprOpAnd::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, state.evalBool(env, e1, pos) && state.evalBool(env, e2, pos)); +} + + +void ExprOpOr::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +} + + +void ExprOpImpl::eval(EvalState & state, Env & env, Value & v) +{ + mkBool(v, !state.evalBool(env, e1, pos) || state.evalBool(env, e2, pos)); +} + + +void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) +{ + Value v1, v2; + state.evalAttrs(env, e1, v1); + state.evalAttrs(env, e2, v2); + + state.nrOpUpdates++; + + if (v1.attrs->size() == 0) { v = v2; return; } + if (v2.attrs->size() == 0) { v = v1; return; } + + state.mkAttrs(v, v1.attrs->size() + v2.attrs->size()); + + /* Merge the sets, preferring values from the second set. Make + sure to keep the resulting vector in sorted order. */ + Bindings::iterator i = v1.attrs->begin(); + Bindings::iterator j = v2.attrs->begin(); + + while (i != v1.attrs->end() && j != v2.attrs->end()) { + if (i->name == j->name) { + v.attrs->push_back(*j); + ++i; ++j; + } + else if (i->name < j->name) + v.attrs->push_back(*i++); + else + v.attrs->push_back(*j++); + } + + while (i != v1.attrs->end()) v.attrs->push_back(*i++); + while (j != v2.attrs->end()) v.attrs->push_back(*j++); + + state.nrOpUpdateValuesCopied += v.attrs->size(); +} + + +void ExprOpConcatLists::eval(EvalState & state, Env & env, Value & v) +{ + Value v1; e1->eval(state, env, v1); + Value v2; e2->eval(state, env, v2); + Value * lists[2] = { &v1, &v2 }; + state.concatLists(v, 2, lists, pos); +} + + +void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos) +{ + nrListConcats++; + + Value * nonEmpty = 0; + size_t len = 0; + for (size_t n = 0; n < nrLists; ++n) { + forceList(*lists[n], pos); + auto l = lists[n]->listSize(); + len += l; + if (l) nonEmpty = lists[n]; + } + + if (nonEmpty && len == nonEmpty->listSize()) { + v = *nonEmpty; + return; + } + + mkList(v, len); + auto out = v.listElems(); + for (size_t n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n]->listSize(); + if (l) + memcpy(out + pos, lists[n]->listElems(), l * sizeof(Value *)); + pos += l; + } +} + + +void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) +{ + PathSet context; + std::ostringstream s; + NixInt n = 0; + NixFloat nf = 0; + + bool first = !forceString; + ValueType firstType = tString; + + for (auto & i : *es) { + Value vTmp; + i->eval(state, env, vTmp); + + /* If the first element is a path, then the result will also + be a path, we don't copy anything (yet - that's done later, + since paths are copied when they are used in a derivation), + and none of the strings are allowed to have contexts. */ + if (first) { + firstType = vTmp.type; + first = false; + } + + if (firstType == tInt) { + if (vTmp.type == tInt) { + n += vTmp.integer; + } else if (vTmp.type == tFloat) { + // Upgrade the type from int to float; + firstType = tFloat; + nf = n; + nf += vTmp.fpoint; + } else + throwEvalError("cannot add %1% to an integer, at %2%", showType(vTmp), pos); + } else if (firstType == tFloat) { + if (vTmp.type == tInt) { + nf += vTmp.integer; + } else if (vTmp.type == tFloat) { + nf += vTmp.fpoint; + } else + throwEvalError("cannot add %1% to a float, at %2%", showType(vTmp), pos); + } else + s << state.coerceToString(pos, vTmp, context, false, firstType == tString); + } + + if (firstType == tInt) + mkInt(v, n); + else if (firstType == tFloat) + mkFloat(v, nf); + else if (firstType == tPath) { + if (!context.empty()) + throwEvalError("a string that refers to a store path cannot be appended to a path, at %1%", pos); + auto path = canonPath(s.str()); + mkPath(v, path.c_str()); + } else + mkString(v, s.str(), context); +} + + +void ExprPos::eval(EvalState & state, Env & env, Value & v) +{ + state.mkPos(v, &pos); +} + + +void EvalState::forceValueDeep(Value & v) +{ + std::set seen; + + std::function recurse; + + recurse = [&](Value & v) { + if (seen.find(&v) != seen.end()) return; + seen.insert(&v); + + forceValue(v); + + if (v.type == tAttrs) { + for (auto & i : *v.attrs) + try { + recurse(*i.value); + } catch (Error & e) { + addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos); + throw; + } + } + + else if (v.isList()) { + for (size_t n = 0; n < v.listSize(); ++n) + recurse(*v.listElems()[n]); + } + }; + + recurse(v); +} + + +NixInt EvalState::forceInt(Value & v, const Pos & pos) +{ + forceValue(v, pos); + if (v.type != tInt) + throwTypeError("value is %1% while an integer was expected, at %2%", v, pos); + return v.integer; +} + + +NixFloat EvalState::forceFloat(Value & v, const Pos & pos) +{ + forceValue(v, pos); + if (v.type == tInt) + return v.integer; + else if (v.type != tFloat) + throwTypeError("value is %1% while a float was expected, at %2%", v, pos); + return v.fpoint; +} + + +bool EvalState::forceBool(Value & v, const Pos & pos) +{ + forceValue(v); + if (v.type != tBool) + throwTypeError("value is %1% while a Boolean was expected, at %2%", v, pos); + return v.boolean; +} + + +bool EvalState::isFunctor(Value & fun) +{ + return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); +} + + +void EvalState::forceFunction(Value & v, const Pos & pos) +{ + forceValue(v); + if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) + throwTypeError("value is %1% while a function was expected, at %2%", v, pos); +} + + +string EvalState::forceString(Value & v, const Pos & pos) +{ + forceValue(v, pos); + if (v.type != tString) { + if (pos) + throwTypeError("value is %1% while a string was expected, at %2%", v, pos); + else + throwTypeError("value is %1% while a string was expected", v); + } + return string(v.string.s); +} + + +void copyContext(const Value & v, PathSet & context) +{ + if (v.string.context) + for (const char * * p = v.string.context; *p; ++p) + context.insert(*p); +} + + +string EvalState::forceString(Value & v, PathSet & context, const Pos & pos) +{ + string s = forceString(v, pos); + copyContext(v, context); + return s; +} + + +string EvalState::forceStringNoCtx(Value & v, const Pos & pos) +{ + string s = forceString(v, pos); + if (v.string.context) { + if (pos) + throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%'), at %3%", + v.string.s, v.string.context[0], pos); + else + throwEvalError("the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string.s, v.string.context[0]); + } + return s; +} + + +bool EvalState::isDerivation(Value & v) +{ + if (v.type != tAttrs) return false; + Bindings::iterator i = v.attrs->find(sType); + if (i == v.attrs->end()) return false; + forceValue(*i->value); + if (i->value->type != tString) return false; + return strcmp(i->value->string.s, "derivation") == 0; +} + + +std::optional EvalState::tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore, bool copyToStore) +{ + auto i = v.attrs->find(sToString); + if (i != v.attrs->end()) { + Value v1; + callFunction(*i->value, v, v1, pos); + return coerceToString(pos, v1, context, coerceMore, copyToStore); + } + + return {}; +} + +string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, + bool coerceMore, bool copyToStore) +{ + forceValue(v); + + string s; + + if (v.type == tString) { + copyContext(v, context); + return v.string.s; + } + + if (v.type == tPath) { + Path path(canonPath(v.path)); + return copyToStore ? copyPathToStore(context, path) : path; + } + + if (v.type == tAttrs) { + auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); + if (maybeString) { + return *maybeString; + } + auto i = v.attrs->find(sOutPath); + if (i == v.attrs->end()) throwTypeError("cannot coerce a set to a string, at %1%", pos); + return coerceToString(pos, *i->value, context, coerceMore, copyToStore); + } + + if (v.type == tExternal) + return v.external->coerceToString(pos, context, coerceMore, copyToStore); + + if (coerceMore) { + + /* Note that `false' is represented as an empty string for + shell scripting convenience, just like `null'. */ + if (v.type == tBool && v.boolean) return "1"; + if (v.type == tBool && !v.boolean) return ""; + if (v.type == tInt) return std::to_string(v.integer); + if (v.type == tFloat) return std::to_string(v.fpoint); + if (v.type == tNull) return ""; + + if (v.isList()) { + string result; + for (size_t n = 0; n < v.listSize(); ++n) { + result += coerceToString(pos, *v.listElems()[n], + context, coerceMore, copyToStore); + if (n < v.listSize() - 1 + /* !!! not quite correct */ + && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) + result += " "; + } + return result; + } + } + + throwTypeError("cannot coerce %1% to a string, at %2%", v, pos); +} + + +string EvalState::copyPathToStore(PathSet & context, const Path & path) +{ + if (nix::isDerivation(path)) + throwEvalError("file names are not allowed to end in '%1%'", drvExtension); + + Path dstPath; + if (srcToStore[path] != "") + dstPath = srcToStore[path]; + else { + dstPath = settings.readOnlyMode + ? store->computeStorePathForPath(baseNameOf(path), checkSourcePath(path)).first + : store->addToStore(baseNameOf(path), checkSourcePath(path), true, htSHA256, defaultPathFilter, repair); + srcToStore[path] = dstPath; + printMsg(lvlChatty, format("copied source '%1%' -> '%2%'") + % path % dstPath); + } + + context.insert(dstPath); + return dstPath; +} + + +Path EvalState::coerceToPath(const Pos & pos, Value & v, PathSet & context) +{ + string path = coerceToString(pos, v, context, false, false); + if (path == "" || path[0] != '/') + throwEvalError("string '%1%' doesn't represent an absolute path, at %2%", path, pos); + return path; +} + + +bool EvalState::eqValues(Value & v1, Value & v2) +{ + forceValue(v1); + forceValue(v2); + + /* !!! Hack to support some old broken code that relies on pointer + equality tests between sets. (Specifically, builderDefs calls + uniqList on a list of sets.) Will remove this eventually. */ + if (&v1 == &v2) return true; + + // Special case type-compatibility between float and int + if (v1.type == tInt && v2.type == tFloat) + return v1.integer == v2.fpoint; + if (v1.type == tFloat && v2.type == tInt) + return v1.fpoint == v2.integer; + + // All other types are not compatible with each other. + if (v1.type != v2.type) return false; + + switch (v1.type) { + + case tInt: + return v1.integer == v2.integer; + + case tBool: + return v1.boolean == v2.boolean; + + case tString: + return strcmp(v1.string.s, v2.string.s) == 0; + + case tPath: + return strcmp(v1.path, v2.path) == 0; + + case tNull: + return true; + + case tList1: + case tList2: + case tListN: + if (v1.listSize() != v2.listSize()) return false; + for (size_t n = 0; n < v1.listSize(); ++n) + if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; + return true; + + case tAttrs: { + /* If both sets denote a derivation (type = "derivation"), + then compare their outPaths. */ + if (isDerivation(v1) && isDerivation(v2)) { + Bindings::iterator i = v1.attrs->find(sOutPath); + Bindings::iterator j = v2.attrs->find(sOutPath); + if (i != v1.attrs->end() && j != v2.attrs->end()) + return eqValues(*i->value, *j->value); + } + + if (v1.attrs->size() != v2.attrs->size()) return false; + + /* Otherwise, compare the attributes one by one. */ + Bindings::iterator i, j; + for (i = v1.attrs->begin(), j = v2.attrs->begin(); i != v1.attrs->end(); ++i, ++j) + if (i->name != j->name || !eqValues(*i->value, *j->value)) + return false; + + return true; + } + + /* Functions are incomparable. */ + case tLambda: + case tPrimOp: + case tPrimOpApp: + return false; + + case tExternal: + return *v1.external == *v2.external; + + case tFloat: + return v1.fpoint == v2.fpoint; + + default: + throwEvalError("cannot compare %1% with %2%", showType(v1), showType(v2)); + } +} + +void EvalState::printStats() +{ + bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; + + struct rusage buf; + getrusage(RUSAGE_SELF, &buf); + float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); + + uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *); + uint64_t bLists = nrListElems * sizeof(Value *); + uint64_t bValues = nrValues * sizeof(Value); + uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); + +#if HAVE_BOEHMGC + GC_word heapSize, totalBytes; + GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); +#endif + if (showStats) { + auto outPath = getEnv("NIX_SHOW_STATS_PATH","-"); + std::fstream fs; + if (outPath != "-") + fs.open(outPath, std::fstream::out); + JSONObject topObj(outPath == "-" ? std::cerr : fs, true); + topObj.attr("cpuTime",cpuTime); + { + auto envs = topObj.object("envs"); + envs.attr("number", nrEnvs); + envs.attr("elements", nrValuesInEnvs); + envs.attr("bytes", bEnvs); + } + { + auto lists = topObj.object("list"); + lists.attr("elements", nrListElems); + lists.attr("bytes", bLists); + lists.attr("concats", nrListConcats); + } + { + auto values = topObj.object("values"); + values.attr("number", nrValues); + values.attr("bytes", bValues); + } + { + auto syms = topObj.object("symbols"); + syms.attr("number", symbols.size()); + syms.attr("bytes", symbols.totalSize()); + } + { + auto sets = topObj.object("sets"); + sets.attr("number", nrAttrsets); + sets.attr("bytes", bAttrsets); + sets.attr("elements", nrAttrsInAttrsets); + } + { + auto sizes = topObj.object("sizes"); + sizes.attr("Env", sizeof(Env)); + sizes.attr("Value", sizeof(Value)); + sizes.attr("Bindings", sizeof(Bindings)); + sizes.attr("Attr", sizeof(Attr)); + } + topObj.attr("nrOpUpdates", nrOpUpdates); + topObj.attr("nrOpUpdateValuesCopied", nrOpUpdateValuesCopied); + topObj.attr("nrThunks", nrThunks); + topObj.attr("nrAvoided", nrAvoided); + topObj.attr("nrLookups", nrLookups); + topObj.attr("nrPrimOpCalls", nrPrimOpCalls); + topObj.attr("nrFunctionCalls", nrFunctionCalls); +#if HAVE_BOEHMGC + { + auto gc = topObj.object("gc"); + gc.attr("heapSize", heapSize); + gc.attr("totalBytes", totalBytes); + } +#endif + + if (countCalls) { + { + auto obj = topObj.object("primops"); + for (auto & i : primOpCalls) + obj.attr(i.first, i.second); + } + { + auto list = topObj.list("functions"); + for (auto & i : functionCalls) { + auto obj = list.object(); + if (i.first->name.set()) + obj.attr("name", (const string &) i.first->name); + else + obj.attr("name", nullptr); + if (i.first->pos) { + obj.attr("file", (const string &) i.first->pos.file); + obj.attr("line", i.first->pos.line); + obj.attr("column", i.first->pos.column); + } + obj.attr("count", i.second); + } + } + { + auto list = topObj.list("attributes"); + for (auto & i : attrSelects) { + auto obj = list.object(); + if (i.first) { + obj.attr("file", (const string &) i.first.file); + obj.attr("line", i.first.line); + obj.attr("column", i.first.column); + } + obj.attr("count", i.second); + } + } + } + + if (getEnv("NIX_SHOW_SYMBOLS", "0") != "0") { + auto list = topObj.list("symbols"); + symbols.dump([&](const std::string & s) { list.elem(s); }); + } + } +} + + +size_t valueSize(Value & v) +{ + std::set seen; + + auto doString = [&](const char * s) -> size_t { + if (seen.find(s) != seen.end()) return 0; + seen.insert(s); + return strlen(s) + 1; + }; + + std::function doValue; + std::function doEnv; + + doValue = [&](Value & v) -> size_t { + if (seen.find(&v) != seen.end()) return 0; + seen.insert(&v); + + size_t sz = sizeof(Value); + + switch (v.type) { + case tString: + sz += doString(v.string.s); + if (v.string.context) + for (const char * * p = v.string.context; *p; ++p) + sz += doString(*p); + break; + case tPath: + sz += doString(v.path); + break; + case tAttrs: + if (seen.find(v.attrs) == seen.end()) { + seen.insert(v.attrs); + sz += sizeof(Bindings) + sizeof(Attr) * v.attrs->capacity(); + for (auto & i : *v.attrs) + sz += doValue(*i.value); + } + break; + case tList1: + case tList2: + case tListN: + if (seen.find(v.listElems()) == seen.end()) { + seen.insert(v.listElems()); + sz += v.listSize() * sizeof(Value *); + for (size_t n = 0; n < v.listSize(); ++n) + sz += doValue(*v.listElems()[n]); + } + break; + case tThunk: + sz += doEnv(*v.thunk.env); + break; + case tApp: + sz += doValue(*v.app.left); + sz += doValue(*v.app.right); + break; + case tLambda: + sz += doEnv(*v.lambda.env); + break; + case tPrimOpApp: + sz += doValue(*v.primOpApp.left); + sz += doValue(*v.primOpApp.right); + break; + case tExternal: + if (seen.find(v.external) != seen.end()) break; + seen.insert(v.external); + sz += v.external->valueSize(seen); + break; + default: + ; + } + + return sz; + }; + + doEnv = [&](Env & env) -> size_t { + if (seen.find(&env) != seen.end()) return 0; + seen.insert(&env); + + size_t sz = sizeof(Env) + sizeof(Value *) * env.size; + + if (env.type != Env::HasWithExpr) + for (size_t i = 0; i < env.size; ++i) + if (env.values[i]) + sz += doValue(*env.values[i]); + + if (env.up) sz += doEnv(*env.up); + + return sz; + }; + + return doValue(v); +} + + +string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +{ + throw TypeError(format("cannot coerce %1% to a string, at %2%") % + showType() % pos); +} + + +bool ExternalValueBase::operator==(const ExternalValueBase & b) const +{ + return false; +} + + +std::ostream & operator << (std::ostream & str, const ExternalValueBase & v) { + return v.print(str); +} + + +EvalSettings evalSettings; + +static GlobalConfig::Register r1(&evalSettings); + + +} diff --git a/third_party/nix/src/libexpr/eval.hh b/third_party/nix/src/libexpr/eval.hh new file mode 100644 index 000000000..8126e4ea5 --- /dev/null +++ b/third_party/nix/src/libexpr/eval.hh @@ -0,0 +1,363 @@ +#pragma once + +#include "attr-set.hh" +#include "value.hh" +#include "nixexpr.hh" +#include "symbol-table.hh" +#include "hash.hh" +#include "config.hh" + +#include +#include +#include + + +namespace nix { + + +class Store; +class EvalState; +enum RepairFlag : bool; + + +typedef void (* PrimOpFun) (EvalState & state, const Pos & pos, Value * * args, Value & v); + + +struct PrimOp +{ + PrimOpFun fun; + size_t arity; + Symbol name; + PrimOp(PrimOpFun fun, size_t arity, Symbol name) + : fun(fun), arity(arity), name(name) { } +}; + + +struct Env +{ + Env * up; + unsigned short size; // used by ‘valueSize’ + unsigned short prevWith:14; // nr of levels up to next `with' environment + enum { Plain = 0, HasWithExpr, HasWithAttrs } type:2; + Value * values[0]; +}; + + +Value & mkString(Value & v, const string & s, const PathSet & context = PathSet()); + +void copyContext(const Value & v, PathSet & context); + + +/* Cache for calls to addToStore(); maps source paths to the store + paths. */ +typedef std::map SrcToStore; + + +std::ostream & operator << (std::ostream & str, const Value & v); + + +typedef std::pair SearchPathElem; +typedef std::list SearchPath; + + +/* Initialise the Boehm GC, if applicable. */ +void initGC(); + + +class EvalState +{ +public: + SymbolTable symbols; + + const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, + sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, + sFile, sLine, sColumn, sFunctor, sToString, + sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sOutputHash, sOutputHashAlgo, sOutputHashMode; + Symbol sDerivationNix; + + /* If set, force copying files to the Nix store even if they + already exist there. */ + RepairFlag repair; + + /* The allowed filesystem paths in restricted or pure evaluation + mode. */ + std::optional allowedPaths; + + Value vEmptySet; + + const ref store; + +private: + SrcToStore srcToStore; + + /* A cache from path names to parse trees. */ +#if HAVE_BOEHMGC + typedef std::map, traceable_allocator > > FileParseCache; +#else + typedef std::map FileParseCache; +#endif + FileParseCache fileParseCache; + + /* A cache from path names to values. */ +#if HAVE_BOEHMGC + typedef std::map, traceable_allocator > > FileEvalCache; +#else + typedef std::map FileEvalCache; +#endif + FileEvalCache fileEvalCache; + + SearchPath searchPath; + + std::map> searchPathResolved; + + /* Cache used by checkSourcePath(). */ + std::unordered_map resolvedPaths; + +public: + + EvalState(const Strings & _searchPath, ref store); + ~EvalState(); + + void addToSearchPath(const string & s); + + SearchPath getSearchPath() { return searchPath; } + + Path checkSourcePath(const Path & path); + + void checkURI(const std::string & uri); + + /* When using a diverted store and 'path' is in the Nix store, map + 'path' to the diverted location (e.g. /nix/store/foo is mapped + to /home/alice/my-nix/nix/store/foo). However, this is only + done if the context is not empty, since otherwise we're + probably trying to read from the actual /nix/store. This is + intended to distinguish between import-from-derivation and + sources stored in the actual /nix/store. */ + Path toRealPath(const Path & path, const PathSet & context); + + /* Parse a Nix expression from the specified file. */ + Expr * parseExprFromFile(const Path & path); + Expr * parseExprFromFile(const Path & path, StaticEnv & staticEnv); + + /* Parse a Nix expression from the specified string. */ + Expr * parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv); + Expr * parseExprFromString(const string & s, const Path & basePath); + + Expr * parseStdin(); + + /* Evaluate an expression read from the given file to normal + form. */ + void evalFile(const Path & path, Value & v); + + void resetFileCache(); + + /* Look up a file in the search path. */ + Path findFile(const string & path); + Path findFile(SearchPath & searchPath, const string & path, const Pos & pos = noPos); + + /* If the specified search path element is a URI, download it. */ + std::pair resolveSearchPathElem(const SearchPathElem & elem); + + /* Evaluate an expression to normal form, storing the result in + value `v'. */ + void eval(Expr * e, Value & v); + + /* Evaluation the expression, then verify that it has the expected + type. */ + inline bool evalBool(Env & env, Expr * e); + inline bool evalBool(Env & env, Expr * e, const Pos & pos); + inline void evalAttrs(Env & env, Expr * e, Value & v); + + /* If `v' is a thunk, enter it and overwrite `v' with the result + of the evaluation of the thunk. If `v' is a delayed function + application, call the function and overwrite `v' with the + result. Otherwise, this is a no-op. */ + inline void forceValue(Value & v, const Pos & pos = noPos); + + /* Force a value, then recursively force list elements and + attributes. */ + void forceValueDeep(Value & v); + + /* Force `v', and then verify that it has the expected type. */ + NixInt forceInt(Value & v, const Pos & pos); + NixFloat forceFloat(Value & v, const Pos & pos); + bool forceBool(Value & v, const Pos & pos); + inline void forceAttrs(Value & v); + inline void forceAttrs(Value & v, const Pos & pos); + inline void forceList(Value & v); + inline void forceList(Value & v, const Pos & pos); + void forceFunction(Value & v, const Pos & pos); // either lambda or primop + string forceString(Value & v, const Pos & pos = noPos); + string forceString(Value & v, PathSet & context, const Pos & pos = noPos); + string forceStringNoCtx(Value & v, const Pos & pos = noPos); + + /* Return true iff the value `v' denotes a derivation (i.e. a + set with attribute `type = "derivation"'). */ + bool isDerivation(Value & v); + + std::optional tryAttrsToString(const Pos & pos, Value & v, + PathSet & context, bool coerceMore = false, bool copyToStore = true); + + /* String coercion. Converts strings, paths and derivations to a + string. If `coerceMore' is set, also converts nulls, integers, + booleans and lists to a string. If `copyToStore' is set, + referenced paths are copied to the Nix store as a side effect. */ + string coerceToString(const Pos & pos, Value & v, PathSet & context, + bool coerceMore = false, bool copyToStore = true); + + string copyPathToStore(PathSet & context, const Path & path); + + /* Path coercion. Converts strings, paths and derivations to a + path. The result is guaranteed to be a canonicalised, absolute + path. Nothing is copied to the store. */ + Path coerceToPath(const Pos & pos, Value & v, PathSet & context); + +public: + + /* The base environment, containing the builtin functions and + values. */ + Env & baseEnv; + + /* The same, but used during parsing to resolve variables. */ + StaticEnv staticBaseEnv; // !!! should be private + +private: + + unsigned int baseEnvDispl = 0; + + void createBaseEnv(); + + Value * addConstant(const string & name, Value & v); + + Value * addPrimOp(const string & name, + size_t arity, PrimOpFun primOp); + +public: + + Value & getBuiltin(const string & name); + +private: + + inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval); + + friend struct ExprVar; + friend struct ExprAttrs; + friend struct ExprLet; + + Expr * parse(const char * text, const Path & path, + const Path & basePath, StaticEnv & staticEnv); + +public: + + /* Do a deep equality test between two values. That is, list + elements and attributes are compared recursively. */ + bool eqValues(Value & v1, Value & v2); + + bool isFunctor(Value & fun); + + void callFunction(Value & fun, Value & arg, Value & v, const Pos & pos); + void callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos); + + /* Automatically call a function for which each argument has a + default value or has a binding in the `args' map. */ + void autoCallFunction(Bindings & args, Value & fun, Value & res); + + /* Allocation primitives. */ + Value * allocValue(); + Env & allocEnv(size_t size); + + Value * allocAttr(Value & vAttrs, const Symbol & name); + + Bindings * allocBindings(size_t capacity); + + void mkList(Value & v, size_t length); + void mkAttrs(Value & v, size_t capacity); + void mkThunk_(Value & v, Expr * expr); + void mkPos(Value & v, Pos * pos); + + void concatLists(Value & v, size_t nrLists, Value * * lists, const Pos & pos); + + /* Print statistics. */ + void printStats(); + + void realiseContext(const PathSet & context); + +private: + + unsigned long nrEnvs = 0; + unsigned long nrValuesInEnvs = 0; + unsigned long nrValues = 0; + unsigned long nrListElems = 0; + unsigned long nrAttrsets = 0; + unsigned long nrAttrsInAttrsets = 0; + unsigned long nrOpUpdates = 0; + unsigned long nrOpUpdateValuesCopied = 0; + unsigned long nrListConcats = 0; + unsigned long nrPrimOpCalls = 0; + unsigned long nrFunctionCalls = 0; + + bool countCalls; + + typedef std::map PrimOpCalls; + PrimOpCalls primOpCalls; + + typedef std::map FunctionCalls; + FunctionCalls functionCalls; + + void incrFunctionCall(ExprLambda * fun); + + typedef std::map AttrSelects; + AttrSelects attrSelects; + + friend struct ExprOpUpdate; + friend struct ExprOpConcatLists; + friend struct ExprSelect; + friend void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v); +}; + + +/* Return a string representing the type of the value `v'. */ +string showType(const Value & v); + +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(const string & s); + +/* If `path' refers to a directory, then append "/default.nix". */ +Path resolveExprPath(Path path); + +struct InvalidPathError : EvalError +{ + Path path; + InvalidPathError(const Path & path); +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~InvalidPathError() throw () { }; +#endif +}; + +struct EvalSettings : Config +{ + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", + "Whether builtin functions that allow executing native code should be enabled."}; + + Setting restrictEval{this, false, "restrict-eval", + "Whether to restrict file system access to paths in $NIX_PATH, " + "and network access to the URI prefixes listed in 'allowed-uris'."}; + + Setting pureEval{this, false, "pure-eval", + "Whether to restrict file system and network access to files specified by cryptographic hash."}; + + Setting enableImportFromDerivation{this, true, "allow-import-from-derivation", + "Whether the evaluator allows importing the result of a derivation."}; + + Setting allowedUris{this, {}, "allowed-uris", + "Prefixes of URIs that builtin functions such as fetchurl and fetchGit are allowed to fetch."}; + + Setting traceFunctionCalls{this, false, "trace-function-calls", + "Emit log messages for each function entry and exit at the 'vomit' log level (-vvvv)"}; +}; + +extern EvalSettings evalSettings; + +} diff --git a/third_party/nix/src/libexpr/function-trace.cc b/third_party/nix/src/libexpr/function-trace.cc new file mode 100644 index 000000000..af1486f78 --- /dev/null +++ b/third_party/nix/src/libexpr/function-trace.cc @@ -0,0 +1,17 @@ +#include "function-trace.hh" + +namespace nix { + +FunctionCallTrace::FunctionCallTrace(const Pos & pos) : pos(pos) { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + printMsg(lvlInfo, "function-trace entered %1% at %2%", pos, ns.count()); +} + +FunctionCallTrace::~FunctionCallTrace() { + auto duration = std::chrono::high_resolution_clock::now().time_since_epoch(); + auto ns = std::chrono::duration_cast(duration); + printMsg(lvlInfo, "function-trace exited %1% at %2%", pos, ns.count()); +} + +} diff --git a/third_party/nix/src/libexpr/function-trace.hh b/third_party/nix/src/libexpr/function-trace.hh new file mode 100644 index 000000000..472f2045e --- /dev/null +++ b/third_party/nix/src/libexpr/function-trace.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "eval.hh" + +#include + +namespace nix { + +struct FunctionCallTrace +{ + const Pos & pos; + FunctionCallTrace(const Pos & pos); + ~FunctionCallTrace(); +}; +} diff --git a/third_party/nix/src/libexpr/get-drvs.cc b/third_party/nix/src/libexpr/get-drvs.cc new file mode 100644 index 000000000..21a4d7917 --- /dev/null +++ b/third_party/nix/src/libexpr/get-drvs.cc @@ -0,0 +1,380 @@ +#include "get-drvs.hh" +#include "util.hh" +#include "eval-inline.hh" +#include "derivations.hh" + +#include +#include + + +namespace nix { + + +DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs) + : state(&state), attrs(attrs), attrPath(attrPath) +{ +} + + +DrvInfo::DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs) + : state(&state), attrs(nullptr), attrPath("") +{ + auto spec = parseDrvPathWithOutputs(drvPathWithOutputs); + + drvPath = spec.first; + + auto drv = store->derivationFromPath(drvPath); + + name = storePathToName(drvPath); + + if (spec.second.size() > 1) + throw Error("building more than one derivation output is not supported, in '%s'", drvPathWithOutputs); + + outputName = + spec.second.empty() + ? get(drv.env, "outputName", "out") + : *spec.second.begin(); + + auto i = drv.outputs.find(outputName); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", drvPath, outputName); + + outPath = i->second.path; +} + + +string DrvInfo::queryName() const +{ + if (name == "" && attrs) { + auto i = attrs->find(state->sName); + if (i == attrs->end()) throw TypeError("derivation name missing"); + name = state->forceStringNoCtx(*i->value); + } + return name; +} + + +string DrvInfo::querySystem() const +{ + if (system == "" && attrs) { + auto i = attrs->find(state->sSystem); + system = i == attrs->end() ? "unknown" : state->forceStringNoCtx(*i->value, *i->pos); + } + return system; +} + + +string DrvInfo::queryDrvPath() const +{ + if (drvPath == "" && attrs) { + Bindings::iterator i = attrs->find(state->sDrvPath); + PathSet context; + drvPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + } + return drvPath; +} + + +string DrvInfo::queryOutPath() const +{ + if (outPath == "" && attrs) { + Bindings::iterator i = attrs->find(state->sOutPath); + PathSet context; + outPath = i != attrs->end() ? state->coerceToPath(*i->pos, *i->value, context) : ""; + } + return outPath; +} + + +DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) +{ + if (outputs.empty()) { + /* Get the ‘outputs’ list. */ + Bindings::iterator i; + if (attrs && (i = attrs->find(state->sOutputs)) != attrs->end()) { + state->forceList(*i->value, *i->pos); + + /* For each output... */ + for (unsigned int j = 0; j < i->value->listSize(); ++j) { + /* Evaluate the corresponding set. */ + string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); + Bindings::iterator out = attrs->find(state->symbols.create(name)); + if (out == attrs->end()) continue; // FIXME: throw error? + state->forceAttrs(*out->value); + + /* And evaluate its ‘outPath’ attribute. */ + Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); + if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? + PathSet context; + outputs[name] = state->coerceToPath(*outPath->pos, *outPath->value, context); + } + } else + outputs["out"] = queryOutPath(); + } + if (!onlyOutputsToInstall || !attrs) + return outputs; + + /* Check for `meta.outputsToInstall` and return `outputs` reduced to that. */ + const Value * outTI = queryMeta("outputsToInstall"); + if (!outTI) return outputs; + const auto errMsg = Error("this derivation has bad 'meta.outputsToInstall'"); + /* ^ this shows during `nix-env -i` right under the bad derivation */ + if (!outTI->isList()) throw errMsg; + Outputs result; + for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { + if ((*i)->type != tString) throw errMsg; + auto out = outputs.find((*i)->string.s); + if (out == outputs.end()) throw errMsg; + result.insert(*out); + } + return result; +} + + +string DrvInfo::queryOutputName() const +{ + if (outputName == "" && attrs) { + Bindings::iterator i = attrs->find(state->sOutputName); + outputName = i != attrs->end() ? state->forceStringNoCtx(*i->value) : ""; + } + return outputName; +} + + +Bindings * DrvInfo::getMeta() +{ + if (meta) return meta; + if (!attrs) return 0; + Bindings::iterator a = attrs->find(state->sMeta); + if (a == attrs->end()) return 0; + state->forceAttrs(*a->value, *a->pos); + meta = a->value->attrs; + return meta; +} + + +StringSet DrvInfo::queryMetaNames() +{ + StringSet res; + if (!getMeta()) return res; + for (auto & i : *meta) + res.insert(i.name); + return res; +} + + +bool DrvInfo::checkMeta(Value & v) +{ + state->forceValue(v); + if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) + if (!checkMeta(*v.listElems()[n])) return false; + return true; + } + else if (v.type == tAttrs) { + Bindings::iterator i = v.attrs->find(state->sOutPath); + if (i != v.attrs->end()) return false; + for (auto & i : *v.attrs) + if (!checkMeta(*i.value)) return false; + return true; + } + else return v.type == tInt || v.type == tBool || v.type == tString || + v.type == tFloat; +} + + +Value * DrvInfo::queryMeta(const string & name) +{ + if (!getMeta()) return 0; + Bindings::iterator a = meta->find(state->symbols.create(name)); + if (a == meta->end() || !checkMeta(*a->value)) return 0; + return a->value; +} + + +string DrvInfo::queryMetaString(const string & name) +{ + Value * v = queryMeta(name); + if (!v || v->type != tString) return ""; + return v->string.s; +} + + +NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) +{ + Value * v = queryMeta(name); + if (!v) return def; + if (v->type == tInt) return v->integer; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + integer meta fields. */ + NixInt n; + if (string2Int(v->string.s, n)) return n; + } + return def; +} + +NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) +{ + Value * v = queryMeta(name); + if (!v) return def; + if (v->type == tFloat) return v->fpoint; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + float meta fields. */ + NixFloat n; + if (string2Float(v->string.s, n)) return n; + } + return def; +} + + +bool DrvInfo::queryMetaBool(const string & name, bool def) +{ + Value * v = queryMeta(name); + if (!v) return def; + if (v->type == tBool) return v->boolean; + if (v->type == tString) { + /* Backwards compatibility with before we had support for + Boolean meta fields. */ + if (strcmp(v->string.s, "true") == 0) return true; + if (strcmp(v->string.s, "false") == 0) return false; + } + return def; +} + + +void DrvInfo::setMeta(const string & name, Value * v) +{ + getMeta(); + Bindings * old = meta; + meta = state->allocBindings(1 + (old ? old->size() : 0)); + Symbol sym = state->symbols.create(name); + if (old) + for (auto i : *old) + if (i.name != sym) + meta->push_back(i); + if (v) meta->push_back(Attr(sym, v)); + meta->sort(); +} + + +/* Cache for already considered attrsets. */ +typedef set Done; + + +/* Evaluate value `v'. If it evaluates to a set of type `derivation', + then put information about it in `drvs' (unless it's already in `done'). + The result boolean indicates whether it makes sense + for the caller to recursively search for derivations in `v'. */ +static bool getDerivation(EvalState & state, Value & v, + const string & attrPath, DrvInfos & drvs, Done & done, + bool ignoreAssertionFailures) +{ + try { + state.forceValue(v); + if (!state.isDerivation(v)) return true; + + /* Remove spurious duplicates (e.g., a set like `rec { x = + derivation {...}; y = x;}'. */ + if (done.find(v.attrs) != done.end()) return false; + done.insert(v.attrs); + + DrvInfo drv(state, attrPath, v.attrs); + + drv.queryName(); + + drvs.push_back(drv); + + return false; + + } catch (AssertionError & e) { + if (ignoreAssertionFailures) return false; + throw; + } +} + + +std::optional getDerivation(EvalState & state, Value & v, + bool ignoreAssertionFailures) +{ + Done done; + DrvInfos drvs; + getDerivation(state, v, "", drvs, done, ignoreAssertionFailures); + if (drvs.size() != 1) return {}; + return std::move(drvs.front()); +} + + +static string addToPath(const string & s1, const string & s2) +{ + return s1.empty() ? s2 : s1 + "." + s2; +} + + +static std::regex attrRegex("[A-Za-z_][A-Za-z0-9-_+]*"); + + +static void getDerivations(EvalState & state, Value & vIn, + const string & pathPrefix, Bindings & autoArgs, + DrvInfos & drvs, Done & done, + bool ignoreAssertionFailures) +{ + Value v; + state.autoCallFunction(autoArgs, vIn, v); + + /* Process the expression. */ + if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; + + else if (v.type == tAttrs) { + + /* !!! undocumented hackery to support combining channels in + nix-env.cc. */ + bool combineChannels = v.attrs->find(state.symbols.create("_combineChannels")) != v.attrs->end(); + + /* Consider the attributes in sorted order to get more + deterministic behaviour in nix-env operations (e.g. when + there are names clashes between derivations, the derivation + bound to the attribute with the "lower" name should take + precedence). */ + for (auto & i : v.attrs->lexicographicOrder()) { + debug("evaluating attribute '%1%'", i->name); + if (!std::regex_match(std::string(i->name), attrRegex)) + continue; + string pathPrefix2 = addToPath(pathPrefix, i->name); + if (combineChannels) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i->value->type == tAttrs) { + Bindings::iterator j = i->value->attrs->find(state.symbols.create("recurseForDerivations")); + if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } + } + } + } + + else if (v.isList()) { + for (unsigned int n = 0; n < v.listSize(); ++n) { + string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); + if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } + } + + else throw TypeError("expression does not evaluate to a derivation (or a set or list of those)"); +} + + +void getDerivations(EvalState & state, Value & v, const string & pathPrefix, + Bindings & autoArgs, DrvInfos & drvs, bool ignoreAssertionFailures) +{ + Done done; + getDerivations(state, v, pathPrefix, autoArgs, drvs, done, ignoreAssertionFailures); +} + + +} diff --git a/third_party/nix/src/libexpr/get-drvs.hh b/third_party/nix/src/libexpr/get-drvs.hh new file mode 100644 index 000000000..d7860fc6a --- /dev/null +++ b/third_party/nix/src/libexpr/get-drvs.hh @@ -0,0 +1,89 @@ +#pragma once + +#include "eval.hh" + +#include +#include + + +namespace nix { + + +struct DrvInfo +{ +public: + typedef std::map Outputs; + +private: + EvalState * state; + + mutable string name; + mutable string system; + mutable string drvPath; + mutable string outPath; + mutable string outputName; + Outputs outputs; + + bool failed = false; // set if we get an AssertionError + + Bindings * attrs = nullptr, * meta = nullptr; + + Bindings * getMeta(); + + bool checkMeta(Value & v); + +public: + string attrPath; /* path towards the derivation */ + + DrvInfo(EvalState & state) : state(&state) { }; + DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs); + DrvInfo(EvalState & state, ref store, const std::string & drvPathWithOutputs); + + string queryName() const; + string querySystem() const; + string queryDrvPath() const; + string queryOutPath() const; + string queryOutputName() const; + /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */ + Outputs queryOutputs(bool onlyOutputsToInstall = false); + + StringSet queryMetaNames(); + Value * queryMeta(const string & name); + string queryMetaString(const string & name); + NixInt queryMetaInt(const string & name, NixInt def); + NixFloat queryMetaFloat(const string & name, NixFloat def); + bool queryMetaBool(const string & name, bool def); + void setMeta(const string & name, Value * v); + + /* + MetaInfo queryMetaInfo(EvalState & state) const; + MetaValue queryMetaInfo(EvalState & state, const string & name) const; + */ + + void setName(const string & s) { name = s; } + void setDrvPath(const string & s) { drvPath = s; } + void setOutPath(const string & s) { outPath = s; } + + void setFailed() { failed = true; }; + bool hasFailed() { return failed; }; +}; + + +#if HAVE_BOEHMGC +typedef list > DrvInfos; +#else +typedef list DrvInfos; +#endif + + +/* If value `v' denotes a derivation, return a DrvInfo object + describing it. Otherwise return nothing. */ +std::optional getDerivation(EvalState & state, + Value & v, bool ignoreAssertionFailures); + +void getDerivations(EvalState & state, Value & v, const string & pathPrefix, + Bindings & autoArgs, DrvInfos & drvs, + bool ignoreAssertionFailures); + + +} diff --git a/third_party/nix/src/libexpr/json-to-value.cc b/third_party/nix/src/libexpr/json-to-value.cc new file mode 100644 index 000000000..8bae986f9 --- /dev/null +++ b/third_party/nix/src/libexpr/json-to-value.cc @@ -0,0 +1,149 @@ +#include "json-to-value.hh" + +#include + +namespace nix { + + +static void skipWhitespace(const char * & s) +{ + while (*s == ' ' || *s == '\t' || *s == '\n' || *s == '\r') s++; +} + + +static string parseJSONString(const char * & s) +{ + string res; + if (*s++ != '"') throw JSONParseError("expected JSON string"); + while (*s != '"') { + if (!*s) throw JSONParseError("got end-of-string in JSON string"); + if (*s == '\\') { + s++; + if (*s == '"') res += '"'; + else if (*s == '\\') res += '\\'; + else if (*s == '/') res += '/'; + else if (*s == '/') res += '/'; + else if (*s == 'b') res += '\b'; + else if (*s == 'f') res += '\f'; + else if (*s == 'n') res += '\n'; + else if (*s == 'r') res += '\r'; + else if (*s == 't') res += '\t'; + else if (*s == 'u') throw JSONParseError("\\u characters in JSON strings are currently not supported"); + else throw JSONParseError("invalid escaped character in JSON string"); + s++; + } else + res += *s++; + } + s++; + return res; +} + + +static void parseJSON(EvalState & state, const char * & s, Value & v) +{ + skipWhitespace(s); + + if (!*s) throw JSONParseError("expected JSON value"); + + if (*s == '[') { + s++; + ValueVector values; + values.reserve(128); + skipWhitespace(s); + while (1) { + if (values.empty() && *s == ']') break; + Value * v2 = state.allocValue(); + parseJSON(state, s, *v2); + values.push_back(v2); + skipWhitespace(s); + if (*s == ']') break; + if (*s != ',') throw JSONParseError("expected ',' or ']' after JSON array element"); + s++; + } + s++; + state.mkList(v, values.size()); + for (size_t n = 0; n < values.size(); ++n) + v.listElems()[n] = values[n]; + } + + else if (*s == '{') { + s++; + ValueMap attrs; + while (1) { + skipWhitespace(s); + if (attrs.empty() && *s == '}') break; + string name = parseJSONString(s); + skipWhitespace(s); + if (*s != ':') throw JSONParseError("expected ':' in JSON object"); + s++; + Value * v2 = state.allocValue(); + parseJSON(state, s, *v2); + attrs[state.symbols.create(name)] = v2; + skipWhitespace(s); + if (*s == '}') break; + if (*s != ',') throw JSONParseError("expected ',' or '}' after JSON member"); + s++; + } + state.mkAttrs(v, attrs.size()); + for (auto & i : attrs) + v.attrs->push_back(Attr(i.first, i.second)); + v.attrs->sort(); + s++; + } + + else if (*s == '"') { + mkString(v, parseJSONString(s)); + } + + else if (isdigit(*s) || *s == '-' || *s == '.' ) { + // Buffer into a string first, then use built-in C++ conversions + std::string tmp_number; + ValueType number_type = tInt; + + while (isdigit(*s) || *s == '-' || *s == '.' || *s == 'e' || *s == 'E') { + if (*s == '.' || *s == 'e' || *s == 'E') + number_type = tFloat; + tmp_number += *s++; + } + + try { + if (number_type == tFloat) + mkFloat(v, stod(tmp_number)); + else + mkInt(v, stol(tmp_number)); + } catch (std::invalid_argument & e) { + throw JSONParseError("invalid JSON number"); + } catch (std::out_of_range & e) { + throw JSONParseError("out-of-range JSON number"); + } + } + + else if (strncmp(s, "true", 4) == 0) { + s += 4; + mkBool(v, true); + } + + else if (strncmp(s, "false", 5) == 0) { + s += 5; + mkBool(v, false); + } + + else if (strncmp(s, "null", 4) == 0) { + s += 4; + mkNull(v); + } + + else throw JSONParseError("unrecognised JSON value"); +} + + +void parseJSON(EvalState & state, const string & s_, Value & v) +{ + const char * s = s_.c_str(); + parseJSON(state, s, v); + skipWhitespace(s); + if (*s) throw JSONParseError(format("expected end-of-string while parsing JSON value: %1%") % s); +} + + +} diff --git a/third_party/nix/src/libexpr/json-to-value.hh b/third_party/nix/src/libexpr/json-to-value.hh new file mode 100644 index 000000000..33f35b16c --- /dev/null +++ b/third_party/nix/src/libexpr/json-to-value.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "eval.hh" + +#include + +namespace nix { + +MakeError(JSONParseError, EvalError) + +void parseJSON(EvalState & state, const string & s, Value & v); + +} diff --git a/third_party/nix/src/libexpr/lexer.l b/third_party/nix/src/libexpr/lexer.l new file mode 100644 index 000000000..c34e5c383 --- /dev/null +++ b/third_party/nix/src/libexpr/lexer.l @@ -0,0 +1,222 @@ +%option reentrant bison-bridge bison-locations +%option noyywrap +%option never-interactive +%option stack +%option nodefault +%option nounput noyy_top_state + + +%s DEFAULT +%x STRING +%x IND_STRING + + +%{ +#include + +#include "nixexpr.hh" +#include "parser-tab.hh" + +using namespace nix; + +namespace nix { + + +static void initLoc(YYLTYPE * loc) +{ + loc->first_line = loc->last_line = 1; + loc->first_column = loc->last_column = 1; +} + + +static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) +{ + loc->first_line = loc->last_line; + loc->first_column = loc->last_column; + + while (len--) { + switch (*s++) { + case '\r': + if (*s == '\n') /* cr/lf */ + s++; + /* fall through */ + case '\n': + ++loc->last_line; + loc->last_column = 1; + break; + default: + ++loc->last_column; + } + } +} + + +static Expr * unescapeStr(SymbolTable & symbols, const char * s, size_t length) +{ + string t; + t.reserve(length); + char c; + while ((c = *s++)) { + if (c == '\\') { + assert(*s); + c = *s++; + if (c == 'n') t += '\n'; + else if (c == 'r') t += '\r'; + else if (c == 't') t += '\t'; + else t += c; + } + else if (c == '\r') { + /* Normalise CR and CR/LF into LF. */ + t += '\n'; + if (*s == '\n') s++; /* cr/lf */ + } + else t += c; + } + return new ExprString(symbols.create(t)); +} + + +} + +#define YY_USER_INIT initLoc(yylloc) +#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); + +#define PUSH_STATE(state) yy_push_state(state, yyscanner) +#define POP_STATE() yy_pop_state(yyscanner) + +%} + + +ANY .|\n +ID [a-zA-Z\_][a-zA-Z0-9\_\'\-]* +INT [0-9]+ +FLOAT (([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)? +PATH [a-zA-Z0-9\.\_\-\+]*(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +HPATH \~(\/[a-zA-Z0-9\.\_\-\+]+)+\/? +SPATH \<[a-zA-Z0-9\.\_\-\+]+(\/[a-zA-Z0-9\.\_\-\+]+)*\> +URI [a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9\%\/\?\:\@\&\=\+\$\,\-\_\.\!\~\*\']+ + + +%% + + +if { return IF; } +then { return THEN; } +else { return ELSE; } +assert { return ASSERT; } +with { return WITH; } +let { return LET; } +in { return IN; } +rec { return REC; } +inherit { return INHERIT; } +or { return OR_KW; } +\.\.\. { return ELLIPSIS; } + +\=\= { return EQ; } +\!\= { return NEQ; } +\<\= { return LEQ; } +\>\= { return GEQ; } +\&\& { return AND; } +\|\| { return OR; } +\-\> { return IMPL; } +\/\/ { return UPDATE; } +\+\+ { return CONCAT; } + +{ID} { yylval->id = strdup(yytext); return ID; } +{INT} { errno = 0; + try { + yylval->n = boost::lexical_cast(yytext); + } catch (const boost::bad_lexical_cast &) { + throw ParseError(format("invalid integer '%1%'") % yytext); + } + return INT; + } +{FLOAT} { errno = 0; + yylval->nf = strtod(yytext, 0); + if (errno != 0) + throw ParseError(format("invalid float '%1%'") % yytext); + return FLOAT; + } + +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } + +\} { /* State INITIAL only exists at the bottom of the stack and is + used as a marker. DEFAULT replaces it everywhere else. + Popping when in INITIAL state causes an empty stack exception, + so don't */ + if (YYSTATE != INITIAL) + POP_STATE(); + return '}'; + } +\{ { PUSH_STATE(DEFAULT); return '{'; } + +\" { PUSH_STATE(STRING); return '"'; } +([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})*\$/\" | +([^\$\"\\]|\$[^\{\"\\]|\\{ANY}|\$\\{ANY})+ { + /* It is impossible to match strings ending with '$' with one + regex because trailing contexts are only valid at the end + of a rule. (A sane but undocumented limitation.) */ + yylval->e = unescapeStr(data->symbols, yytext, yyleng); + return STR; + } +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +\" { POP_STATE(); return '"'; } +\$|\\|\$\\ { + /* This can only occur when we reach EOF, otherwise the above + (...|\$[^\{\"\\]|\\.|\$\\.)+ would have triggered. + This is technically invalid, but we leave the problem to the + parser who fails with exact location. */ + return STR; + } + +\'\'(\ *\n)? { PUSH_STATE(IND_STRING); return IND_STRING_OPEN; } +([^\$\']|\$[^\{\']|\'[^\'\$])+ { + yylval->e = new ExprIndStr(yytext); + return IND_STR; + } +\'\'\$ | +\$ { + yylval->e = new ExprIndStr("$"); + return IND_STR; + } +\'\'\' { + yylval->e = new ExprIndStr("''"); + return IND_STR; + } +\'\'\\{ANY} { + yylval->e = unescapeStr(data->symbols, yytext + 2, yyleng - 2); + return IND_STR; + } +\$\{ { PUSH_STATE(DEFAULT); return DOLLAR_CURLY; } +\'\' { POP_STATE(); return IND_STRING_CLOSE; } +\' { + yylval->e = new ExprIndStr("'"); + return IND_STR; + } + + +{PATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path '%s' has a trailing slash", yytext); + yylval->path = strdup(yytext); + return PATH; + } +{HPATH} { if (yytext[yyleng-1] == '/') + throw ParseError("path '%s' has a trailing slash", yytext); + yylval->path = strdup(yytext); + return HPATH; + } +{SPATH} { yylval->path = strdup(yytext); return SPATH; } +{URI} { yylval->uri = strdup(yytext); return URI; } + +[ \t\r\n]+ /* eat up whitespace */ +\#[^\r\n]* /* single-line comments */ +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ + +{ANY} { + /* Don't return a negative number, as this will cause + Bison to stop parsing without an error. */ + return (unsigned char) yytext[0]; + } + +%% + diff --git a/third_party/nix/src/libexpr/local.mk b/third_party/nix/src/libexpr/local.mk new file mode 100644 index 000000000..ccd5293e4 --- /dev/null +++ b/third_party/nix/src/libexpr/local.mk @@ -0,0 +1,33 @@ +libraries += libexpr + +libexpr_NAME = libnixexpr + +libexpr_DIR := $(d) + +libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc + +libexpr_LIBS = libutil libstore + +libexpr_LDFLAGS = +ifneq ($(OS), FreeBSD) + libexpr_LDFLAGS += -ldl +endif + +# The dependency on libgc must be propagated (i.e. meaning that +# programs/libraries that use libexpr must explicitly pass -lgc), +# because inline functions in libexpr's header files call libgc. +libexpr_LDFLAGS_PROPAGATED = $(BDW_GC_LIBS) + +libexpr_ORDER_AFTER := $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh + +$(d)/parser-tab.cc $(d)/parser-tab.hh: $(d)/parser.y + $(trace-gen) bison -v -o $(libexpr_DIR)/parser-tab.cc $< -d + +$(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l + $(trace-gen) flex --outfile $(libexpr_DIR)/lexer-tab.cc --header-file=$(libexpr_DIR)/lexer-tab.hh $< + +clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh + +dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh + +$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) diff --git a/third_party/nix/src/libexpr/names.cc b/third_party/nix/src/libexpr/names.cc new file mode 100644 index 000000000..382088c78 --- /dev/null +++ b/third_party/nix/src/libexpr/names.cc @@ -0,0 +1,107 @@ +#include "names.hh" +#include "util.hh" + + +namespace nix { + + +DrvName::DrvName() +{ + name = ""; +} + + +/* Parse a derivation name. The `name' part of a derivation name is + everything up to but not including the first dash *not* followed by + a letter. The `version' part is the rest (excluding the separating + dash). E.g., `apache-httpd-2.0.48' is parsed to (`apache-httpd', + '2.0.48'). */ +DrvName::DrvName(const string & s) : hits(0) +{ + name = fullName = s; + for (unsigned int i = 0; i < s.size(); ++i) { + /* !!! isalpha/isdigit are affected by the locale. */ + if (s[i] == '-' && i + 1 < s.size() && !isalpha(s[i + 1])) { + name = string(s, 0, i); + version = string(s, i + 1); + break; + } + } +} + + +bool DrvName::matches(DrvName & n) +{ + if (name != "*") { + if (!regex) regex = std::unique_ptr(new std::regex(name, std::regex::extended)); + if (!std::regex_match(n.name, *regex)) return false; + } + if (version != "" && version != n.version) return false; + return true; +} + + +string nextComponent(string::const_iterator & p, + const string::const_iterator end) +{ + /* Skip any dots and dashes (component separators). */ + while (p != end && (*p == '.' || *p == '-')) ++p; + + if (p == end) return ""; + + /* If the first character is a digit, consume the longest sequence + of digits. Otherwise, consume the longest sequence of + non-digit, non-separator characters. */ + string s; + if (isdigit(*p)) + while (p != end && isdigit(*p)) s += *p++; + else + while (p != end && (!isdigit(*p) && *p != '.' && *p != '-')) + s += *p++; + + return s; +} + + +static bool componentsLT(const string & c1, const string & c2) +{ + int n1, n2; + bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + + if (c1Num && c2Num) return n1 < n2; + else if (c1 == "" && c2Num) return true; + else if (c1 == "pre" && c2 != "pre") return true; + else if (c2 == "pre") return false; + /* Assume that `2.3a' < `2.3.1'. */ + else if (c2Num) return true; + else if (c1Num) return false; + else return c1 < c2; +} + + +int compareVersions(const string & v1, const string & v2) +{ + string::const_iterator p1 = v1.begin(); + string::const_iterator p2 = v2.begin(); + + while (p1 != v1.end() || p2 != v2.end()) { + string c1 = nextComponent(p1, v1.end()); + string c2 = nextComponent(p2, v2.end()); + if (componentsLT(c1, c2)) return -1; + else if (componentsLT(c2, c1)) return 1; + } + + return 0; +} + + +DrvNames drvNamesFromArgs(const Strings & opArgs) +{ + DrvNames result; + for (auto & i : opArgs) + result.push_back(DrvName(i)); + return result; +} + + +} diff --git a/third_party/nix/src/libexpr/names.hh b/third_party/nix/src/libexpr/names.hh new file mode 100644 index 000000000..13c3093e7 --- /dev/null +++ b/third_party/nix/src/libexpr/names.hh @@ -0,0 +1,32 @@ +#pragma once + +#include + +#include "types.hh" +#include + +namespace nix { + +struct DrvName +{ + string fullName; + string name; + string version; + unsigned int hits; + + DrvName(); + DrvName(const string & s); + bool matches(DrvName & n); + +private: + std::unique_ptr regex; +}; + +typedef list DrvNames; + +string nextComponent(string::const_iterator & p, + const string::const_iterator end); +int compareVersions(const string & v1, const string & v2); +DrvNames drvNamesFromArgs(const Strings & opArgs); + +} diff --git a/third_party/nix/src/libexpr/nix-expr.pc.in b/third_party/nix/src/libexpr/nix-expr.pc.in new file mode 100644 index 000000000..80f7a492b --- /dev/null +++ b/third_party/nix/src/libexpr/nix-expr.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-store bdw-gc +Libs: -L${libdir} -lnixexpr +Cflags: -I${includedir}/nix -std=c++17 diff --git a/third_party/nix/src/libexpr/nixexpr.cc b/third_party/nix/src/libexpr/nixexpr.cc new file mode 100644 index 000000000..63cbef1dd --- /dev/null +++ b/third_party/nix/src/libexpr/nixexpr.cc @@ -0,0 +1,438 @@ +#include "nixexpr.hh" +#include "derivations.hh" +#include "util.hh" + +#include + + +namespace nix { + + +/* Displaying abstract syntax trees. */ + +std::ostream & operator << (std::ostream & str, const Expr & e) +{ + e.show(str); + return str; +} + +static void showString(std::ostream & str, const string & s) +{ + str << '"'; + for (auto c : (string) s) + if (c == '"' || c == '\\' || c == '$') str << "\\" << c; + else if (c == '\n') str << "\\n"; + else if (c == '\r') str << "\\r"; + else if (c == '\t') str << "\\t"; + else str << c; + str << '"'; +} + +static void showId(std::ostream & str, const string & s) +{ + if (s.empty()) + str << "\"\""; + else if (s == "if") // FIXME: handle other keywords + str << '"' << s << '"'; + else { + char c = s[0]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_')) { + showString(str, s); + return; + } + for (auto c : s) + if (!((c >= 'a' && c <= 'z') || + (c >= 'A' && c <= 'Z') || + (c >= '0' && c <= '9') || + c == '_' || c == '\'' || c == '-')) { + showString(str, s); + return; + } + str << s; + } +} + +std::ostream & operator << (std::ostream & str, const Symbol & sym) +{ + showId(str, *sym.s); + return str; +} + +void Expr::show(std::ostream & str) const +{ + abort(); +} + +void ExprInt::show(std::ostream & str) const +{ + str << n; +} + +void ExprFloat::show(std::ostream & str) const +{ + str << nf; +} + +void ExprString::show(std::ostream & str) const +{ + showString(str, s); +} + +void ExprPath::show(std::ostream & str) const +{ + str << s; +} + +void ExprVar::show(std::ostream & str) const +{ + str << name; +} + +void ExprSelect::show(std::ostream & str) const +{ + str << "(" << *e << ")." << showAttrPath(attrPath); + if (def) str << " or (" << *def << ")"; +} + +void ExprOpHasAttr::show(std::ostream & str) const +{ + str << "((" << *e << ") ? " << showAttrPath(attrPath) << ")"; +} + +void ExprAttrs::show(std::ostream & str) const +{ + if (recursive) str << "rec "; + str << "{ "; + for (auto & i : attrs) + if (i.second.inherited) + str << "inherit " << i.first << " " << "; "; + else + str << i.first << " = " << *i.second.e << "; "; + for (auto & i : dynamicAttrs) + str << "\"${" << *i.nameExpr << "}\" = " << *i.valueExpr << "; "; + str << "}"; +} + +void ExprList::show(std::ostream & str) const +{ + str << "[ "; + for (auto & i : elems) + str << "(" << *i << ") "; + str << "]"; +} + +void ExprLambda::show(std::ostream & str) const +{ + str << "("; + if (matchAttrs) { + str << "{ "; + bool first = true; + for (auto & i : formals->formals) { + if (first) first = false; else str << ", "; + str << i.name; + if (i.def) str << " ? " << *i.def; + } + if (formals->ellipsis) { + if (!first) str << ", "; + str << "..."; + } + str << " }"; + if (!arg.empty()) str << " @ "; + } + if (!arg.empty()) str << arg; + str << ": " << *body << ")"; +} + +void ExprLet::show(std::ostream & str) const +{ + str << "(let "; + for (auto & i : attrs->attrs) + if (i.second.inherited) { + str << "inherit " << i.first << "; "; + } + else + str << i.first << " = " << *i.second.e << "; "; + str << "in " << *body << ")"; +} + +void ExprWith::show(std::ostream & str) const +{ + str << "(with " << *attrs << "; " << *body << ")"; +} + +void ExprIf::show(std::ostream & str) const +{ + str << "(if " << *cond << " then " << *then << " else " << *else_ << ")"; +} + +void ExprAssert::show(std::ostream & str) const +{ + str << "assert " << *cond << "; " << *body; +} + +void ExprOpNot::show(std::ostream & str) const +{ + str << "(! " << *e << ")"; +} + +void ExprConcatStrings::show(std::ostream & str) const +{ + bool first = true; + str << "("; + for (auto & i : *es) { + if (first) first = false; else str << " + "; + str << *i; + } + str << ")"; +} + +void ExprPos::show(std::ostream & str) const +{ + str << "__curPos"; +} + + +std::ostream & operator << (std::ostream & str, const Pos & pos) +{ + if (!pos) + str << "undefined position"; + else + str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str(); + return str; +} + + +string showAttrPath(const AttrPath & attrPath) +{ + std::ostringstream out; + bool first = true; + for (auto & i : attrPath) { + if (!first) out << '.'; else first = false; + if (i.symbol.set()) + out << i.symbol; + else + out << "\"${" << *i.expr << "}\""; + } + return out.str(); +} + + +Pos noPos; + + +/* Computing levels/displacements for variables. */ + +void Expr::bindVars(const StaticEnv & env) +{ + abort(); +} + +void ExprInt::bindVars(const StaticEnv & env) +{ +} + +void ExprFloat::bindVars(const StaticEnv & env) +{ +} + +void ExprString::bindVars(const StaticEnv & env) +{ +} + +void ExprPath::bindVars(const StaticEnv & env) +{ +} + +void ExprVar::bindVars(const StaticEnv & env) +{ + /* Check whether the variable appears in the environment. If so, + set its level and displacement. */ + const StaticEnv * curEnv; + unsigned int level; + int withLevel = -1; + for (curEnv = &env, level = 0; curEnv; curEnv = curEnv->up, level++) { + if (curEnv->isWith) { + if (withLevel == -1) withLevel = level; + } else { + StaticEnv::Vars::const_iterator i = curEnv->vars.find(name); + if (i != curEnv->vars.end()) { + fromWith = false; + this->level = level; + displ = i->second; + return; + } + } + } + + /* Otherwise, the variable must be obtained from the nearest + enclosing `with'. If there is no `with', then we can issue an + "undefined variable" error now. */ + if (withLevel == -1) throw UndefinedVarError(format("undefined variable '%1%' at %2%") % name % pos); + + fromWith = true; + this->level = withLevel; +} + +void ExprSelect::bindVars(const StaticEnv & env) +{ + e->bindVars(env); + if (def) def->bindVars(env); + for (auto & i : attrPath) + if (!i.symbol.set()) + i.expr->bindVars(env); +} + +void ExprOpHasAttr::bindVars(const StaticEnv & env) +{ + e->bindVars(env); + for (auto & i : attrPath) + if (!i.symbol.set()) + i.expr->bindVars(env); +} + +void ExprAttrs::bindVars(const StaticEnv & env) +{ + const StaticEnv * dynamicEnv = &env; + StaticEnv newEnv(false, &env); + + if (recursive) { + dynamicEnv = &newEnv; + + unsigned int displ = 0; + for (auto & i : attrs) + newEnv.vars[i.first] = i.second.displ = displ++; + + for (auto & i : attrs) + i.second.e->bindVars(i.second.inherited ? env : newEnv); + } + + else + for (auto & i : attrs) + i.second.e->bindVars(env); + + for (auto & i : dynamicAttrs) { + i.nameExpr->bindVars(*dynamicEnv); + i.valueExpr->bindVars(*dynamicEnv); + } +} + +void ExprList::bindVars(const StaticEnv & env) +{ + for (auto & i : elems) + i->bindVars(env); +} + +void ExprLambda::bindVars(const StaticEnv & env) +{ + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + + if (!arg.empty()) newEnv.vars[arg] = displ++; + + if (matchAttrs) { + for (auto & i : formals->formals) + newEnv.vars[i.name] = displ++; + + for (auto & i : formals->formals) + if (i.def) i.def->bindVars(newEnv); + } + + body->bindVars(newEnv); +} + +void ExprLet::bindVars(const StaticEnv & env) +{ + StaticEnv newEnv(false, &env); + + unsigned int displ = 0; + for (auto & i : attrs->attrs) + newEnv.vars[i.first] = i.second.displ = displ++; + + for (auto & i : attrs->attrs) + i.second.e->bindVars(i.second.inherited ? env : newEnv); + + body->bindVars(newEnv); +} + +void ExprWith::bindVars(const StaticEnv & env) +{ + /* Does this `with' have an enclosing `with'? If so, record its + level so that `lookupVar' can look up variables in the previous + `with' if this one doesn't contain the desired attribute. */ + const StaticEnv * curEnv; + unsigned int level; + prevWith = 0; + for (curEnv = &env, level = 1; curEnv; curEnv = curEnv->up, level++) + if (curEnv->isWith) { + prevWith = level; + break; + } + + attrs->bindVars(env); + StaticEnv newEnv(true, &env); + body->bindVars(newEnv); +} + +void ExprIf::bindVars(const StaticEnv & env) +{ + cond->bindVars(env); + then->bindVars(env); + else_->bindVars(env); +} + +void ExprAssert::bindVars(const StaticEnv & env) +{ + cond->bindVars(env); + body->bindVars(env); +} + +void ExprOpNot::bindVars(const StaticEnv & env) +{ + e->bindVars(env); +} + +void ExprConcatStrings::bindVars(const StaticEnv & env) +{ + for (auto & i : *es) + i->bindVars(env); +} + +void ExprPos::bindVars(const StaticEnv & env) +{ +} + + +/* Storing function names. */ + +void Expr::setName(Symbol & name) +{ +} + + +void ExprLambda::setName(Symbol & name) +{ + this->name = name; + body->setName(name); +} + + +string ExprLambda::showNamePos() const +{ + return (format("%1% at %2%") % (name.set() ? "'" + (string) name + "'" : "anonymous function") % pos).str(); +} + + + +/* Symbol table. */ + +size_t SymbolTable::totalSize() const +{ + size_t n = 0; + for (auto & i : symbols) + n += i.size(); + return n; +} + + +} diff --git a/third_party/nix/src/libexpr/nixexpr.hh b/third_party/nix/src/libexpr/nixexpr.hh new file mode 100644 index 000000000..665a42987 --- /dev/null +++ b/third_party/nix/src/libexpr/nixexpr.hh @@ -0,0 +1,342 @@ +#pragma once + +#include "value.hh" +#include "symbol-table.hh" + +#include + + +namespace nix { + + +MakeError(EvalError, Error) +MakeError(ParseError, Error) +MakeError(AssertionError, EvalError) +MakeError(ThrownError, AssertionError) +MakeError(Abort, EvalError) +MakeError(TypeError, EvalError) +MakeError(UndefinedVarError, Error) +MakeError(RestrictedPathError, Error) + + +/* Position objects. */ + +struct Pos +{ + Symbol file; + unsigned int line, column; + Pos() : line(0), column(0) { }; + Pos(const Symbol & file, unsigned int line, unsigned int column) + : file(file), line(line), column(column) { }; + operator bool() const + { + return line != 0; + } + bool operator < (const Pos & p2) const + { + if (!line) return p2.line; + if (!p2.line) return false; + int d = ((string) file).compare((string) p2.file); + if (d < 0) return true; + if (d > 0) return false; + if (line < p2.line) return true; + if (line > p2.line) return false; + return column < p2.column; + } +}; + +extern Pos noPos; + +std::ostream & operator << (std::ostream & str, const Pos & pos); + + +struct Env; +struct Value; +class EvalState; +struct StaticEnv; + + +/* An attribute path is a sequence of attribute names. */ +struct AttrName +{ + Symbol symbol; + Expr * expr; + AttrName(const Symbol & s) : symbol(s) {}; + AttrName(Expr * e) : expr(e) {}; +}; + +typedef std::vector AttrPath; + +string showAttrPath(const AttrPath & attrPath); + + +/* Abstract syntax of Nix expressions. */ + +struct Expr +{ + virtual ~Expr() { }; + virtual void show(std::ostream & str) const; + virtual void bindVars(const StaticEnv & env); + virtual void eval(EvalState & state, Env & env, Value & v); + virtual Value * maybeThunk(EvalState & state, Env & env); + virtual void setName(Symbol & name); +}; + +std::ostream & operator << (std::ostream & str, const Expr & e); + +#define COMMON_METHODS \ + void show(std::ostream & str) const; \ + void eval(EvalState & state, Env & env, Value & v); \ + void bindVars(const StaticEnv & env); + +struct ExprInt : Expr +{ + NixInt n; + Value v; + ExprInt(NixInt n) : n(n) { mkInt(v, n); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprFloat : Expr +{ + NixFloat nf; + Value v; + ExprFloat(NixFloat nf) : nf(nf) { mkFloat(v, nf); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprString : Expr +{ + Symbol s; + Value v; + ExprString(const Symbol & s) : s(s) { mkString(v, s); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +/* Temporary class used during parsing of indented strings. */ +struct ExprIndStr : Expr +{ + string s; + ExprIndStr(const string & s) : s(s) { }; +}; + +struct ExprPath : Expr +{ + string s; + Value v; + ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprVar : Expr +{ + Pos pos; + Symbol name; + + /* Whether the variable comes from an environment (e.g. a rec, let + or function argument) or from a "with". */ + bool fromWith; + + /* In the former case, the value is obtained by going `level' + levels up from the current environment and getting the + `displ'th value in that environment. In the latter case, the + value is obtained by getting the attribute named `name' from + the set stored in the environment that is `level' levels up + from the current one.*/ + unsigned int level; + unsigned int displ; + + ExprVar(const Symbol & name) : name(name) { }; + ExprVar(const Pos & pos, const Symbol & name) : pos(pos), name(name) { }; + COMMON_METHODS + Value * maybeThunk(EvalState & state, Env & env); +}; + +struct ExprSelect : Expr +{ + Pos pos; + Expr * e, * def; + AttrPath attrPath; + ExprSelect(const Pos & pos, Expr * e, const AttrPath & attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(attrPath) { }; + ExprSelect(const Pos & pos, Expr * e, const Symbol & name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; + COMMON_METHODS +}; + +struct ExprOpHasAttr : Expr +{ + Expr * e; + AttrPath attrPath; + ExprOpHasAttr(Expr * e, const AttrPath & attrPath) : e(e), attrPath(attrPath) { }; + COMMON_METHODS +}; + +struct ExprAttrs : Expr +{ + bool recursive; + struct AttrDef { + bool inherited; + Expr * e; + Pos pos; + unsigned int displ; // displacement + AttrDef(Expr * e, const Pos & pos, bool inherited=false) + : inherited(inherited), e(e), pos(pos) { }; + AttrDef() { }; + }; + typedef std::map AttrDefs; + AttrDefs attrs; + struct DynamicAttrDef { + Expr * nameExpr, * valueExpr; + Pos pos; + DynamicAttrDef(Expr * nameExpr, Expr * valueExpr, const Pos & pos) + : nameExpr(nameExpr), valueExpr(valueExpr), pos(pos) { }; + }; + typedef std::vector DynamicAttrDefs; + DynamicAttrDefs dynamicAttrs; + ExprAttrs() : recursive(false) { }; + COMMON_METHODS +}; + +struct ExprList : Expr +{ + std::vector elems; + ExprList() { }; + COMMON_METHODS +}; + +struct Formal +{ + Symbol name; + Expr * def; + Formal(const Symbol & name, Expr * def) : name(name), def(def) { }; +}; + +struct Formals +{ + typedef std::list Formals_; + Formals_ formals; + std::set argNames; // used during parsing + bool ellipsis; +}; + +struct ExprLambda : Expr +{ + Pos pos; + Symbol name; + Symbol arg; + bool matchAttrs; + Formals * formals; + Expr * body; + ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body) + : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body) + { + if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end()) + throw ParseError(format("duplicate formal function argument '%1%' at %2%") + % arg % pos); + }; + void setName(Symbol & name); + string showNamePos() const; + COMMON_METHODS +}; + +struct ExprLet : Expr +{ + ExprAttrs * attrs; + Expr * body; + ExprLet(ExprAttrs * attrs, Expr * body) : attrs(attrs), body(body) { }; + COMMON_METHODS +}; + +struct ExprWith : Expr +{ + Pos pos; + Expr * attrs, * body; + size_t prevWith; + ExprWith(const Pos & pos, Expr * attrs, Expr * body) : pos(pos), attrs(attrs), body(body) { }; + COMMON_METHODS +}; + +struct ExprIf : Expr +{ + Expr * cond, * then, * else_; + ExprIf(Expr * cond, Expr * then, Expr * else_) : cond(cond), then(then), else_(else_) { }; + COMMON_METHODS +}; + +struct ExprAssert : Expr +{ + Pos pos; + Expr * cond, * body; + ExprAssert(const Pos & pos, Expr * cond, Expr * body) : pos(pos), cond(cond), body(body) { }; + COMMON_METHODS +}; + +struct ExprOpNot : Expr +{ + Expr * e; + ExprOpNot(Expr * e) : e(e) { }; + COMMON_METHODS +}; + +#define MakeBinOp(name, s) \ + struct name : Expr \ + { \ + Pos pos; \ + Expr * e1, * e2; \ + name(Expr * e1, Expr * e2) : e1(e1), e2(e2) { }; \ + name(const Pos & pos, Expr * e1, Expr * e2) : pos(pos), e1(e1), e2(e2) { }; \ + void show(std::ostream & str) const \ + { \ + str << "(" << *e1 << " " s " " << *e2 << ")"; \ + } \ + void bindVars(const StaticEnv & env) \ + { \ + e1->bindVars(env); e2->bindVars(env); \ + } \ + void eval(EvalState & state, Env & env, Value & v); \ + }; + +MakeBinOp(ExprApp, "") +MakeBinOp(ExprOpEq, "==") +MakeBinOp(ExprOpNEq, "!=") +MakeBinOp(ExprOpAnd, "&&") +MakeBinOp(ExprOpOr, "||") +MakeBinOp(ExprOpImpl, "->") +MakeBinOp(ExprOpUpdate, "//") +MakeBinOp(ExprOpConcatLists, "++") + +struct ExprConcatStrings : Expr +{ + Pos pos; + bool forceString; + vector * es; + ExprConcatStrings(const Pos & pos, bool forceString, vector * es) + : pos(pos), forceString(forceString), es(es) { }; + COMMON_METHODS +}; + +struct ExprPos : Expr +{ + Pos pos; + ExprPos(const Pos & pos) : pos(pos) { }; + COMMON_METHODS +}; + + +/* Static environments are used to map variable names onto (level, + displacement) pairs used to obtain the value of the variable at + runtime. */ +struct StaticEnv +{ + bool isWith; + const StaticEnv * up; + typedef std::map Vars; + Vars vars; + StaticEnv(bool isWith, const StaticEnv * up) : isWith(isWith), up(up) { }; +}; + + +} diff --git a/third_party/nix/src/libexpr/parser.y b/third_party/nix/src/libexpr/parser.y new file mode 100644 index 000000000..967c88d9b --- /dev/null +++ b/third_party/nix/src/libexpr/parser.y @@ -0,0 +1,704 @@ +%glr-parser +%pure-parser +%locations +%define parse.error verbose +%defines +/* %no-lines */ +%parse-param { void * scanner } +%parse-param { nix::ParseData * data } +%lex-param { void * scanner } +%lex-param { nix::ParseData * data } +%expect 1 +%expect-rr 1 + +%code requires { + +#ifndef BISON_HEADER +#define BISON_HEADER + +#include "util.hh" + +#include "nixexpr.hh" +#include "eval.hh" + +namespace nix { + + struct ParseData + { + EvalState & state; + SymbolTable & symbols; + Expr * result; + Path basePath; + Symbol path; + string error; + Symbol sLetBody; + ParseData(EvalState & state) + : state(state) + , symbols(state.symbols) + , sLetBody(symbols.create("")) + { }; + }; + +} + +#define YY_DECL int yylex \ + (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParseData * data) + +#endif + +} + +%{ + +#include "parser-tab.hh" +#include "lexer-tab.hh" + +YY_DECL; + +using namespace nix; + + +namespace nix { + + +static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos) +{ + throw ParseError(format("attribute '%1%' at %2% already defined at %3%") + % showAttrPath(attrPath) % pos % prevPos); +} + + +static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos) +{ + throw ParseError(format("attribute '%1%' at %2% already defined at %3%") + % attr % pos % prevPos); +} + + +static void addAttr(ExprAttrs * attrs, AttrPath & attrPath, + Expr * e, const Pos & pos) +{ + AttrPath::iterator i; + // All attrpaths have at least one attr + assert(!attrPath.empty()); + // Checking attrPath validity. + // =========================== + for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { + if (i->symbol.set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + if (!j->second.inherited) { + ExprAttrs * attrs2 = dynamic_cast(j->second.e); + if (!attrs2) dupAttr(attrPath, pos, j->second.pos); + attrs = attrs2; + } else + dupAttr(attrPath, pos, j->second.pos); + } else { + ExprAttrs * nested = new ExprAttrs; + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(nested, pos); + attrs = nested; + } + } else { + ExprAttrs *nested = new ExprAttrs; + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, nested, pos)); + attrs = nested; + } + } + // Expr insertion. + // ========================== + if (i->symbol.set()) { + ExprAttrs::AttrDefs::iterator j = attrs->attrs.find(i->symbol); + if (j != attrs->attrs.end()) { + // This attr path is already defined. However, if both + // e and the expr pointed by the attr path are two attribute sets, + // we want to merge them. + // Otherwise, throw an error. + auto ae = dynamic_cast(e); + auto jAttrs = dynamic_cast(j->second.e); + if (jAttrs && ae) { + for (auto & ad : ae->attrs) { + auto j2 = jAttrs->attrs.find(ad.first); + if (j2 != jAttrs->attrs.end()) // Attr already defined in iAttrs, error. + dupAttr(ad.first, j2->second.pos, ad.second.pos); + jAttrs->attrs[ad.first] = ad.second; + } + } else { + dupAttr(attrPath, pos, j->second.pos); + } + } else { + // This attr path is not defined. Let's create it. + attrs->attrs[i->symbol] = ExprAttrs::AttrDef(e, pos); + e->setName(i->symbol); + } + } else { + attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); + } +} + + +static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) +{ + if (formals->argNames.find(formal.name) != formals->argNames.end()) + throw ParseError(format("duplicate formal function argument '%1%' at %2%") + % formal.name % pos); + formals->formals.push_front(formal); + formals->argNames.insert(formal.name); +} + + +static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector & es) +{ + if (es.empty()) return new ExprString(symbols.create("")); + + /* Figure out the minimum indentation. Note that by design + whitespace-only final lines are not taken into account. (So + the " " in "\n ''" is ignored, but the " " in "\n foo''" is.) */ + bool atStartOfLine = true; /* = seen only whitespace in the current line */ + size_t minIndent = 1000000; + size_t curIndent = 0; + for (auto & i : es) { + ExprIndStr * e = dynamic_cast(i); + if (!e) { + /* Anti-quotations end the current start-of-line whitespace. */ + if (atStartOfLine) { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + continue; + } + for (size_t j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') + curIndent++; + else if (e->s[j] == '\n') { + /* Empty line, doesn't influence minimum + indentation. */ + curIndent = 0; + } else { + atStartOfLine = false; + if (curIndent < minIndent) minIndent = curIndent; + } + } else if (e->s[j] == '\n') { + atStartOfLine = true; + curIndent = 0; + } + } + } + + /* Strip spaces from each line. */ + vector * es2 = new vector; + atStartOfLine = true; + size_t curDropped = 0; + size_t n = es.size(); + for (vector::iterator i = es.begin(); i != es.end(); ++i, --n) { + ExprIndStr * e = dynamic_cast(*i); + if (!e) { + atStartOfLine = false; + curDropped = 0; + es2->push_back(*i); + continue; + } + + string s2; + for (size_t j = 0; j < e->s.size(); ++j) { + if (atStartOfLine) { + if (e->s[j] == ' ') { + if (curDropped++ >= minIndent) + s2 += e->s[j]; + } + else if (e->s[j] == '\n') { + curDropped = 0; + s2 += e->s[j]; + } else { + atStartOfLine = false; + curDropped = 0; + s2 += e->s[j]; + } + } else { + s2 += e->s[j]; + if (e->s[j] == '\n') atStartOfLine = true; + } + } + + /* Remove the last line if it is empty and consists only of + spaces. */ + if (n == 1) { + string::size_type p = s2.find_last_of('\n'); + if (p != string::npos && s2.find_first_not_of(' ', p + 1) == string::npos) + s2 = string(s2, 0, p + 1); + } + + es2->push_back(new ExprString(symbols.create(s2))); + } + + /* If this is a single string, then don't do a concatenation. */ + return es2->size() == 1 && dynamic_cast((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2); +} + + +static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data) +{ + return Pos(data->path, loc.first_line, loc.first_column); +} + +#define CUR_POS makeCurPos(*yylocp, data) + + +} + + +void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error) +{ + data->error = (format("%1%, at %2%") + % error % makeCurPos(*loc, data)).str(); +} + + +%} + +%union { + // !!! We're probably leaking stuff here. + nix::Expr * e; + nix::ExprList * list; + nix::ExprAttrs * attrs; + nix::Formals * formals; + nix::Formal * formal; + nix::NixInt n; + nix::NixFloat nf; + const char * id; // !!! -> Symbol + char * path; + char * uri; + std::vector * attrNames; + std::vector * string_parts; +} + +%type start expr expr_function expr_if expr_op +%type expr_app expr_select expr_simple +%type expr_list +%type binds +%type formals +%type formal +%type attrs attrpath +%type string_parts_interpolated ind_string_parts +%type string_parts string_attr +%type attr +%token ID ATTRPATH +%token STR IND_STR +%token INT +%token FLOAT +%token PATH HPATH SPATH +%token URI +%token IF THEN ELSE ASSERT WITH LET IN REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token DOLLAR_CURLY /* == ${ */ +%token IND_STRING_OPEN IND_STRING_CLOSE +%token ELLIPSIS + +%right IMPL +%left OR +%left AND +%nonassoc EQ NEQ +%nonassoc '<' '>' LEQ GEQ +%right UPDATE +%left NOT +%left '+' '-' +%left '*' '/' +%right CONCAT +%nonassoc '?' +%nonassoc NEGATE + +%% + +start: expr { data->result = $1; }; + +expr: expr_function; + +expr_function + : ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); } + | '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); } + | '{' formals '}' '@' ID ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); } + | ID '@' '{' formals '}' ':' expr_function + { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); } + | ASSERT expr ';' expr_function + { $$ = new ExprAssert(CUR_POS, $2, $4); } + | WITH expr ';' expr_function + { $$ = new ExprWith(CUR_POS, $2, $4); } + | LET binds IN expr_function + { if (!$2->dynamicAttrs.empty()) + throw ParseError(format("dynamic attributes not allowed in let at %1%") + % CUR_POS); + $$ = new ExprLet($2, $4); + } + | expr_if + ; + +expr_if + : IF expr THEN expr ELSE expr { $$ = new ExprIf($2, $4, $6); } + | expr_op + ; + +expr_op + : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } + | '-' expr_op %prec NEGATE { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), new ExprInt(0)), $2); } + | expr_op EQ expr_op { $$ = new ExprOpEq($1, $3); } + | expr_op NEQ expr_op { $$ = new ExprOpNEq($1, $3); } + | expr_op '<' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3); } + | expr_op LEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1)); } + | expr_op '>' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $3), $1); } + | expr_op GEQ expr_op { $$ = new ExprOpNot(new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__lessThan")), $1), $3)); } + | expr_op AND expr_op { $$ = new ExprOpAnd(CUR_POS, $1, $3); } + | expr_op OR expr_op { $$ = new ExprOpOr(CUR_POS, $1, $3); } + | expr_op IMPL expr_op { $$ = new ExprOpImpl(CUR_POS, $1, $3); } + | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } + | expr_op '+' expr_op + { $$ = new ExprConcatStrings(CUR_POS, false, new vector({$1, $3})); } + | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } + | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } + | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } + | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } + | expr_app + ; + +expr_app + : expr_app expr_select + { $$ = new ExprApp(CUR_POS, $1, $2); } + | expr_select { $$ = $1; } + ; + +expr_select + : expr_simple '.' attrpath + { $$ = new ExprSelect(CUR_POS, $1, *$3, 0); } + | expr_simple '.' attrpath OR_KW expr_select + { $$ = new ExprSelect(CUR_POS, $1, *$3, $5); } + | /* Backwards compatibility: because Nixpkgs has a rarely used + function named ‘or’, allow stuff like ‘map or [...]’. */ + expr_simple OR_KW + { $$ = new ExprApp(CUR_POS, $1, new ExprVar(CUR_POS, data->symbols.create("or"))); } + | expr_simple { $$ = $1; } + ; + +expr_simple + : ID { + if (strcmp($1, "__curPos") == 0) + $$ = new ExprPos(CUR_POS); + else + $$ = new ExprVar(CUR_POS, data->symbols.create($1)); + } + | INT { $$ = new ExprInt($1); } + | FLOAT { $$ = new ExprFloat($1); } + | '"' string_parts '"' { $$ = $2; } + | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { + $$ = stripIndentation(CUR_POS, data->symbols, *$2); + } + | PATH { $$ = new ExprPath(absPath($1, data->basePath)); } + | HPATH { $$ = new ExprPath(getHome() + string{$1 + 1}); } + | SPATH { + string path($1 + 1, strlen($1) - 2); + $$ = new ExprApp(CUR_POS, + new ExprApp(new ExprVar(data->symbols.create("__findFile")), + new ExprVar(data->symbols.create("__nixPath"))), + new ExprString(data->symbols.create(path))); + } + | URI { $$ = new ExprString(data->symbols.create($1)); } + | '(' expr ')' { $$ = $2; } + /* Let expressions `let {..., body = ...}' are just desugared + into `(rec {..., body = ...}).body'. */ + | LET '{' binds '}' + { $3->recursive = true; $$ = new ExprSelect(noPos, $3, data->symbols.create("body")); } + | REC '{' binds '}' + { $3->recursive = true; $$ = $3; } + | '{' binds '}' + { $$ = $2; } + | '[' expr_list ']' { $$ = $2; } + ; + +string_parts + : STR + | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); } + | { $$ = new ExprString(data->symbols.create("")); } + ; + +string_parts_interpolated + : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | DOLLAR_CURLY expr '}' { $$ = new vector; $$->push_back($2); } + | STR DOLLAR_CURLY expr '}' { + $$ = new vector; + $$->push_back($1); + $$->push_back($3); + } + ; + +ind_string_parts + : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } + | { $$ = new vector; } + ; + +binds + : binds attrpath '=' expr ';' { $$ = $1; addAttr($$, *$2, $4, makeCurPos(@2, data)); } + | binds INHERIT attrs ';' + { $$ = $1; + for (auto & i : *$3) { + if ($$->attrs.find(i.symbol) != $$->attrs.end()) + dupAttr(i.symbol, makeCurPos(@3, data), $$->attrs[i.symbol].pos); + Pos pos = makeCurPos(@3, data); + $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprVar(CUR_POS, i.symbol), pos, true); + } + } + | binds INHERIT '(' expr ')' attrs ';' + { $$ = $1; + /* !!! Should ensure sharing of the expression in $4. */ + for (auto & i : *$6) { + if ($$->attrs.find(i.symbol) != $$->attrs.end()) + dupAttr(i.symbol, makeCurPos(@6, data), $$->attrs[i.symbol].pos); + $$->attrs[i.symbol] = ExprAttrs::AttrDef(new ExprSelect(CUR_POS, $4, i.symbol), makeCurPos(@6, data)); + } + } + | { $$ = new ExprAttrs; } + ; + +attrs + : attrs attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($2))); } + | attrs string_attr + { $$ = $1; + ExprString * str = dynamic_cast($2); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + throw ParseError(format("dynamic attributes not allowed in inherit at %1%") + % makeCurPos(@2, data)); + } + | { $$ = new AttrPath; } + ; + +attrpath + : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(data->symbols.create($3))); } + | attrpath '.' string_attr + { $$ = $1; + ExprString * str = dynamic_cast($3); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + $$->push_back(AttrName($3)); + } + | attr { $$ = new vector; $$->push_back(AttrName(data->symbols.create($1))); } + | string_attr + { $$ = new vector; + ExprString *str = dynamic_cast($1); + if (str) { + $$->push_back(AttrName(str->s)); + delete str; + } else + $$->push_back(AttrName($1)); + } + ; + +attr + : ID { $$ = $1; } + | OR_KW { $$ = "or"; } + ; + +string_attr + : '"' string_parts '"' { $$ = $2; } + | DOLLAR_CURLY expr '}' { $$ = $2; } + ; + +expr_list + : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ } + | { $$ = new ExprList; } + ; + +formals + : formal ',' formals + { $$ = $3; addFormal(CUR_POS, $$, *$1); } + | formal + { $$ = new Formals; addFormal(CUR_POS, $$, *$1); $$->ellipsis = false; } + | + { $$ = new Formals; $$->ellipsis = false; } + | ELLIPSIS + { $$ = new Formals; $$->ellipsis = true; } + ; + +formal + : ID { $$ = new Formal(data->symbols.create($1), 0); } + | ID '?' expr { $$ = new Formal(data->symbols.create($1), $3); } + ; + +%% + + +#include +#include +#include +#include + +#include "eval.hh" +#include "download.hh" +#include "store-api.hh" + + +namespace nix { + + +Expr * EvalState::parse(const char * text, + const Path & path, const Path & basePath, StaticEnv & staticEnv) +{ + yyscan_t scanner; + ParseData data(*this); + data.basePath = basePath; + data.path = data.symbols.create(path); + + yylex_init(&scanner); + yy_scan_string(text, scanner); + int res = yyparse(scanner, &data); + yylex_destroy(scanner); + + if (res) throw ParseError(data.error); + + data.result->bindVars(staticEnv); + + return data.result; +} + + +Path resolveExprPath(Path path) +{ + assert(path[0] == '/'); + + /* If `path' is a symlink, follow it. This is so that relative + path references work. */ + struct stat st; + while (true) { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of '%1%'") % path); + if (!S_ISLNK(st.st_mode)) break; + path = absPath(readLink(path), dirOf(path)); + } + + /* If `path' refers to a directory, append `/default.nix'. */ + if (S_ISDIR(st.st_mode)) + path = canonPath(path + "/default.nix"); + + return path; +} + + +Expr * EvalState::parseExprFromFile(const Path & path) +{ + return parseExprFromFile(path, staticBaseEnv); +} + + +Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv) +{ + return parse(readFile(path).c_str(), path, dirOf(path), staticEnv); +} + + +Expr * EvalState::parseExprFromString(const string & s, const Path & basePath, StaticEnv & staticEnv) +{ + return parse(s.c_str(), "(string)", basePath, staticEnv); +} + + +Expr * EvalState::parseExprFromString(const string & s, const Path & basePath) +{ + return parseExprFromString(s, basePath, staticBaseEnv); +} + + +Expr * EvalState::parseStdin() +{ + //Activity act(*logger, lvlTalkative, format("parsing standard input")); + return parseExprFromString(drainFD(0), absPath(".")); +} + + +void EvalState::addToSearchPath(const string & s) +{ + size_t pos = s.find('='); + string prefix; + Path path; + if (pos == string::npos) { + path = s; + } else { + prefix = string(s, 0, pos); + path = string(s, pos + 1); + } + + searchPath.emplace_back(prefix, path); +} + + +Path EvalState::findFile(const string & path) +{ + return findFile(searchPath, path); +} + + +Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos & pos) +{ + for (auto & i : searchPath) { + std::string suffix; + if (i.first.empty()) + suffix = "/" + path; + else { + auto s = i.first.size(); + if (path.compare(0, s, i.first) != 0 || + (path.size() > s && path[s] != '/')) + continue; + suffix = path.size() == s ? "" : "/" + string(path, s); + } + auto r = resolveSearchPathElem(i); + if (!r.first) continue; + Path res = r.second + suffix; + if (pathExists(res)) return canonPath(res); + } + format f = format( + "file '%1%' was not found in the Nix search path (add it using $NIX_PATH or -I)" + + string(pos ? ", at %2%" : "")); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + throw ThrownError(f % path % pos); +} + + +std::pair EvalState::resolveSearchPathElem(const SearchPathElem & elem) +{ + auto i = searchPathResolved.find(elem.second); + if (i != searchPathResolved.end()) return i->second; + + std::pair res; + + if (isUri(elem.second)) { + try { + CachedDownloadRequest request(elem.second); + request.unpack = true; + res = { true, getDownloader()->downloadCached(store, request).path }; + } catch (DownloadError & e) { + printError(format("warning: Nix search path entry '%1%' cannot be downloaded, ignoring") % elem.second); + res = { false, "" }; + } + } else { + auto path = absPath(elem.second); + if (pathExists(path)) + res = { true, path }; + else { + printError(format("warning: Nix search path entry '%1%' does not exist, ignoring") % elem.second); + res = { false, "" }; + } + } + + debug(format("resolved search path element '%s' to '%s'") % elem.second % res.second); + + searchPathResolved[elem.second] = res; + return res; +} + + +} diff --git a/third_party/nix/src/libexpr/primops.cc b/third_party/nix/src/libexpr/primops.cc new file mode 100644 index 000000000..d4c60f870 --- /dev/null +++ b/third_party/nix/src/libexpr/primops.cc @@ -0,0 +1,2329 @@ +#include "archive.hh" +#include "derivations.hh" +#include "download.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "globals.hh" +#include "json-to-value.hh" +#include "names.hh" +#include "store-api.hh" +#include "util.hh" +#include "json.hh" +#include "value-to-json.hh" +#include "value-to-xml.hh" +#include "primops.hh" + +#include +#include +#include + +#include +#include +#include +#include + + +namespace nix { + + +/************************************************************* + * Miscellaneous + *************************************************************/ + + +/* Decode a context string ‘!!’ into a pair . */ +std::pair decodeContext(const string & s) +{ + if (s.at(0) == '!') { + size_t index = s.find("!", 1); + return std::pair(string(s, index + 1), string(s, 1, index - 1)); + } else + return std::pair(s.at(0) == '/' ? s : string(s, 1), ""); +} + + +InvalidPathError::InvalidPathError(const Path & path) : + EvalError(format("path '%1%' is not valid") % path), path(path) {} + +void EvalState::realiseContext(const PathSet & context) +{ + PathSet drvs; + + for (auto & i : context) { + std::pair decoded = decodeContext(i); + Path ctx = decoded.first; + assert(store->isStorePath(ctx)); + if (!store->isValidPath(ctx)) + throw InvalidPathError(ctx); + if (!decoded.second.empty() && nix::isDerivation(ctx)) { + drvs.insert(decoded.first + "!" + decoded.second); + + /* Add the output of this derivation to the allowed + paths. */ + if (allowedPaths) { + auto drv = store->derivationFromPath(decoded.first); + DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + allowedPaths->insert(i->second.path); + } + } + } + + if (drvs.empty()) return; + + if (!evalSettings.enableImportFromDerivation) + throw EvalError(format("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false") % *(drvs.begin())); + + /* For performance, prefetch all substitute info. */ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); +} + + +/* Load and evaluate an expression from path specified by the + argument. */ +static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + + Path realPath = state.checkSourcePath(state.toRealPath(path, context)); + + if (state.store->isStorePath(path) && state.store->isValidPath(path) && isDerivation(path)) { + Derivation drv = readDerivation(realPath); + Value & w = *state.allocValue(); + state.mkAttrs(w, 3 + drv.outputs.size()); + Value * v2 = state.allocAttr(w, state.sDrvPath); + mkString(*v2, path, {"=" + path}); + v2 = state.allocAttr(w, state.sName); + mkString(*v2, drv.env["name"]); + Value * outputsVal = + state.allocAttr(w, state.symbols.create("outputs")); + state.mkList(*outputsVal, drv.outputs.size()); + unsigned int outputs_index = 0; + + for (const auto & o : drv.outputs) { + v2 = state.allocAttr(w, state.symbols.create(o.first)); + mkString(*v2, o.second.path, {"!" + o.first + "!" + path}); + outputsVal->listElems()[outputs_index] = state.allocValue(); + mkString(*(outputsVal->listElems()[outputs_index++]), o.first); + } + w.attrs->sort(); + Value fun; + state.evalFile(settings.nixDataDir + "/nix/corepkgs/imported-drv-to-derivation.nix", fun); + state.forceFunction(fun, pos); + mkApp(v, fun, w); + state.forceAttrs(v, pos); + } else { + state.forceAttrs(*args[0]); + if (args[0]->attrs->empty()) + state.evalFile(realPath, v); + else { + Env * env = &state.allocEnv(args[0]->attrs->size()); + env->up = &state.baseEnv; + + StaticEnv staticEnv(false, &state.staticBaseEnv); + + unsigned int displ = 0; + for (auto & attr : *args[0]->attrs) { + staticEnv.vars[attr.name] = displ; + env->values[displ++] = attr.value; + } + + printTalkative("evaluating file '%1%'", realPath); + Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + + e->eval(state, *env, v); + } + } +} + + +/* Want reasonable symbol names, so extern C */ +/* !!! Should we pass the Pos or the file name too? */ +extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); + +/* Load a ValueInitializer from a DSO and return whatever it initializes */ +void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot import '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + + path = state.checkSourcePath(path); + + string sym = state.forceStringNoCtx(*args[1], pos); + + void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw EvalError(format("could not open '%1%': %2%") % path % dlerror()); + + dlerror(); + ValueInitializer func = (ValueInitializer) dlsym(handle, sym.c_str()); + if(!func) { + char *message = dlerror(); + if (message) + throw EvalError(format("could not load symbol '%1%' from '%2%': %3%") % sym % path % message); + else + throw EvalError(format("symbol '%1%' from '%2%' resolved to NULL when a function pointer was expected") + % sym % path); + } + + (func)(state, v); + + /* We don't dlclose because v may be a primop referencing a function in the shared object file */ +} + + +/* Execute a program and parse its output */ +void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + auto elems = args[0]->listElems(); + auto count = args[0]->listSize(); + if (count == 0) { + throw EvalError(format("at least one argument to 'exec' required, at %1%") % pos); + } + PathSet context; + auto program = state.coerceToString(pos, *elems[0], context, false, false); + Strings commandArgs; + for (unsigned int i = 1; i < args[0]->listSize(); ++i) { + commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); + } + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot execute '%1%', since path '%2%' is not valid, at %3%") + % program % e.path % pos); + } + + auto output = runProgram(program, true, commandArgs); + Expr * parsed; + try { + parsed = state.parseExprFromString(output, pos.file); + } catch (Error & e) { + e.addPrefix(format("While parsing the output from '%1%', at %2%\n") % program % pos); + throw; + } + try { + state.eval(parsed, v); + } catch (Error & e) { + e.addPrefix(format("While evaluating the output from '%1%', at %2%\n") % program % pos); + throw; + } +} + + +/* Return a string representing the type of the expression. */ +static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + string t; + switch (args[0]->type) { + case tInt: t = "int"; break; + case tBool: t = "bool"; break; + case tString: t = "string"; break; + case tPath: t = "path"; break; + case tNull: t = "null"; break; + case tAttrs: t = "set"; break; + case tList1: case tList2: case tListN: t = "list"; break; + case tLambda: + case tPrimOp: + case tPrimOpApp: + t = "lambda"; + break; + case tExternal: + t = args[0]->external->typeOf(); + break; + case tFloat: t = "float"; break; + default: abort(); + } + mkString(v, state.symbols.create(t)); +} + + +/* Determine whether the argument is the null value. */ +static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tNull); +} + + +/* Determine whether the argument is a function. */ +static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + bool res; + switch (args[0]->type) { + case tLambda: + case tPrimOp: + case tPrimOpApp: + res = true; + break; + default: + res = false; + break; + } + mkBool(v, res); +} + + +/* Determine whether the argument is an integer. */ +static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tInt); +} + +/* Determine whether the argument is a float. */ +static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tFloat); +} + +/* Determine whether the argument is a string. */ +static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tString); +} + + +/* Determine whether the argument is a Boolean. */ +static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tBool); +} + +/* Determine whether the argument is a path. */ +static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tPath); +} + +struct CompareValues +{ + bool operator () (const Value * v1, const Value * v2) const + { + if (v1->type == tFloat && v2->type == tInt) + return v1->fpoint < v2->integer; + if (v1->type == tInt && v2->type == tFloat) + return v1->integer < v2->fpoint; + if (v1->type != v2->type) + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + switch (v1->type) { + case tInt: + return v1->integer < v2->integer; + case tFloat: + return v1->fpoint < v2->fpoint; + case tString: + return strcmp(v1->string.s, v2->string.s) < 0; + case tPath: + return strcmp(v1->path, v2->path) < 0; + default: + throw EvalError(format("cannot compare %1% with %2%") % showType(*v1) % showType(*v2)); + } + } +}; + + +#if HAVE_BOEHMGC +typedef list > ValueList; +#else +typedef list ValueList; +#endif + + +static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + + /* Get the start set. */ + Bindings::iterator startSet = + args[0]->attrs->find(state.symbols.create("startSet")); + if (startSet == args[0]->attrs->end()) + throw EvalError(format("attribute 'startSet' required, at %1%") % pos); + state.forceList(*startSet->value, pos); + + ValueList workSet; + for (unsigned int n = 0; n < startSet->value->listSize(); ++n) + workSet.push_back(startSet->value->listElems()[n]); + + /* Get the operator. */ + Bindings::iterator op = + args[0]->attrs->find(state.symbols.create("operator")); + if (op == args[0]->attrs->end()) + throw EvalError(format("attribute 'operator' required, at %1%") % pos); + state.forceValue(*op->value); + + /* Construct the closure by applying the operator to element of + `workSet', adding the result to `workSet', continuing until + no new elements are found. */ + ValueList res; + // `doneKeys' doesn't need to be a GC root, because its values are + // reachable from res. + set doneKeys; + while (!workSet.empty()) { + Value * e = *(workSet.begin()); + workSet.pop_front(); + + state.forceAttrs(*e, pos); + + Bindings::iterator key = + e->attrs->find(state.symbols.create("key")); + if (key == e->attrs->end()) + throw EvalError(format("attribute 'key' required, at %1%") % pos); + state.forceValue(*key->value); + + if (doneKeys.find(key->value) != doneKeys.end()) continue; + doneKeys.insert(key->value); + res.push_back(e); + + /* Call the `operator' function with `e' as argument. */ + Value call; + mkApp(call, *op->value, *e); + state.forceList(call, pos); + + /* Add the values returned by the operator to the work set. */ + for (unsigned int n = 0; n < call.listSize(); ++n) { + state.forceValue(*call.listElems()[n]); + workSet.push_back(call.listElems()[n]); + } + } + + /* Create the result list. */ + state.mkList(v, res.size()); + unsigned int n = 0; + for (auto & i : res) + v.listElems()[n++] = i; +} + + +static void prim_abort(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw Abort(format("evaluation aborted with the following error message: '%1%'") % s); +} + + +static void prim_throw(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + throw ThrownError(s); +} + + +static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + try { + state.forceValue(*args[1]); + v = *args[1]; + } catch (Error & e) { + PathSet context; + e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context)); + throw; + } +} + + +/* Try evaluating the argument. Success => {success=true; value=something;}, + * else => {success=false; value=false;} */ +static void prim_tryEval(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.mkAttrs(v, 2); + try { + state.forceValue(*args[0]); + v.attrs->push_back(Attr(state.sValue, args[0])); + mkBool(*state.allocAttr(v, state.symbols.create("success")), true); + } catch (AssertionError & e) { + mkBool(*state.allocAttr(v, state.sValue), false); + mkBool(*state.allocAttr(v, state.symbols.create("success")), false); + } + v.attrs->sort(); +} + + +/* Return an environment variable. Use with care. */ +static void prim_getEnv(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string name = state.forceStringNoCtx(*args[0], pos); + mkString(v, evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name)); +} + + +/* Evaluate the first argument, then return the second argument. */ +static void prim_seq(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + + +/* Evaluate the first argument deeply (i.e. recursing into lists and + attrsets), then return the second argument. */ +static void prim_deepSeq(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValueDeep(*args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + + +/* Evaluate the first expression and print it on standard error. Then + return the second expression. Useful for debugging. */ +static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + if (args[0]->type == tString) + printError(format("trace: %1%") % args[0]->string.s); + else + printError(format("trace: %1%") % *args[0]); + state.forceValue(*args[1]); + v = *args[1]; +} + + +void prim_valueSize(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + /* We're not forcing the argument on purpose. */ + mkInt(v, valueSize(*args[0])); +} + + +/************************************************************* + * Derivations + *************************************************************/ + + +/* Construct (as a unobservable side effect) a Nix derivation + expression that performs the derivation described by the argument + set. Returns the original set extended with the following + attributes: `outPath' containing the primary output path of the + derivation; `drvPath' containing the path of the Nix expression; + and `type' set to `derivation' to indicate that this is a + derivation. */ +static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + + /* Figure out the name first (for stack backtraces). */ + Bindings::iterator attr = args[0]->attrs->find(state.sName); + if (attr == args[0]->attrs->end()) + throw EvalError(format("required attribute 'name' missing, at %1%") % pos); + string drvName; + Pos & posDrvName(*attr->pos); + try { + drvName = state.forceStringNoCtx(*attr->value, pos); + } catch (Error & e) { + e.addPrefix(format("while evaluating the derivation attribute 'name' at %1%:\n") % posDrvName); + throw; + } + + /* Check whether attributes should be passed as a JSON file. */ + std::ostringstream jsonBuf; + std::unique_ptr jsonObject; + attr = args[0]->attrs->find(state.sStructuredAttrs); + if (attr != args[0]->attrs->end() && state.forceBool(*attr->value, pos)) + jsonObject = std::make_unique(jsonBuf); + + /* Check whether null attributes should be ignored. */ + bool ignoreNulls = false; + attr = args[0]->attrs->find(state.sIgnoreNulls); + if (attr != args[0]->attrs->end()) + ignoreNulls = state.forceBool(*attr->value, pos); + + /* Build the derivation expression by processing the attributes. */ + Derivation drv; + + PathSet context; + + std::optional outputHash; + std::string outputHashAlgo; + bool outputHashRecursive = false; + + StringSet outputs; + outputs.insert("out"); + + for (auto & i : args[0]->attrs->lexicographicOrder()) { + if (i->name == state.sIgnoreNulls) continue; + const string & key = i->name; + vomit("processing attribute '%1%'", key); + + auto handleHashMode = [&](const std::string & s) { + if (s == "recursive") outputHashRecursive = true; + else if (s == "flat") outputHashRecursive = false; + else throw EvalError("invalid value '%s' for 'outputHashMode' attribute, at %s", s, posDrvName); + }; + + auto handleOutputs = [&](const Strings & ss) { + outputs.clear(); + for (auto & j : ss) { + if (outputs.find(j) != outputs.end()) + throw EvalError(format("duplicate derivation output '%1%', at %2%") % j % posDrvName); + /* !!! Check whether j is a valid attribute + name. */ + /* Derivations cannot be named ‘drv’, because + then we'd have an attribute ‘drvPath’ in + the resulting set. */ + if (j == "drv") + throw EvalError(format("invalid derivation output name 'drv', at %1%") % posDrvName); + outputs.insert(j); + } + if (outputs.empty()) + throw EvalError(format("derivation cannot have an empty set of outputs, at %1%") % posDrvName); + }; + + try { + + if (ignoreNulls) { + state.forceValue(*i->value); + if (i->value->type == tNull) continue; + } + + /* The `args' attribute is special: it supplies the + command-line arguments to the builder. */ + if (i->name == state.sArgs) { + state.forceList(*i->value, pos); + for (unsigned int n = 0; n < i->value->listSize(); ++n) { + string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); + drv.args.push_back(s); + } + } + + /* All other attributes are passed to the builder through + the environment. */ + else { + + if (jsonObject) { + + if (i->name == state.sStructuredAttrs) continue; + + auto placeholder(jsonObject->placeholder(key)); + printValueAsJSON(state, true, *i->value, placeholder, context); + + if (i->name == state.sBuilder) + drv.builder = state.forceString(*i->value, context, posDrvName); + else if (i->name == state.sSystem) + drv.platform = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHash) + outputHash = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHashAlgo) + outputHashAlgo = state.forceStringNoCtx(*i->value, posDrvName); + else if (i->name == state.sOutputHashMode) + handleHashMode(state.forceStringNoCtx(*i->value, posDrvName)); + else if (i->name == state.sOutputs) { + /* Require ‘outputs’ to be a list of strings. */ + state.forceList(*i->value, posDrvName); + Strings ss; + for (unsigned int n = 0; n < i->value->listSize(); ++n) + ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + handleOutputs(ss); + } + + } else { + auto s = state.coerceToString(posDrvName, *i->value, context, true); + drv.env.emplace(key, s); + if (i->name == state.sBuilder) drv.builder = s; + else if (i->name == state.sSystem) drv.platform = s; + else if (i->name == state.sOutputHash) outputHash = s; + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = s; + else if (i->name == state.sOutputHashMode) handleHashMode(s); + else if (i->name == state.sOutputs) + handleOutputs(tokenizeString(s)); + } + + } + + } catch (Error & e) { + e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n") + % key % drvName % posDrvName); + throw; + } + } + + if (jsonObject) { + jsonObject.reset(); + drv.env.emplace("__json", jsonBuf.str()); + } + + /* Everything in the context of the strings in the derivation + attributes should be added as dependencies of the resulting + derivation. */ + for (auto & path : context) { + + /* Paths marked with `=' denote that the path of a derivation + is explicitly passed to the builder. Since that allows the + builder to gain access to every path in the dependency + graph of the derivation (including all outputs), all paths + in the graph must be added to this derivation's list of + inputs to ensure that they are available when the builder + runs. */ + if (path.at(0) == '=') { + /* !!! This doesn't work if readOnlyMode is set. */ + PathSet refs; + state.store->computeFSClosure(string(path, 1), refs); + for (auto & j : refs) { + drv.inputSrcs.insert(j); + if (isDerivation(j)) + drv.inputDrvs[j] = state.store->queryDerivationOutputNames(j); + } + } + + /* Handle derivation outputs of the form ‘!!’. */ + else if (path.at(0) == '!') { + std::pair ctx = decodeContext(path); + drv.inputDrvs[ctx.first].insert(ctx.second); + } + + /* Otherwise it's a source file. */ + else + drv.inputSrcs.insert(path); + } + + /* Do we have all required attributes? */ + if (drv.builder == "") + throw EvalError(format("required attribute 'builder' missing, at %1%") % posDrvName); + if (drv.platform == "") + throw EvalError(format("required attribute 'system' missing, at %1%") % posDrvName); + + /* Check whether the derivation name is valid. */ + checkStoreName(drvName); + if (isDerivation(drvName)) + throw EvalError(format("derivation names are not allowed to end in '%1%', at %2%") + % drvExtension % posDrvName); + + if (outputHash) { + /* Handle fixed-output derivations. */ + if (outputs.size() != 1 || *(outputs.begin()) != "out") + throw Error(format("multiple outputs are not supported in fixed-output derivations, at %1%") % posDrvName); + + HashType ht = outputHashAlgo.empty() ? htUnknown : parseHashType(outputHashAlgo); + Hash h(*outputHash, ht); + + Path outPath = state.store->makeFixedOutputPath(outputHashRecursive, h, drvName); + if (!jsonObject) drv.env["out"] = outPath; + drv.outputs["out"] = DerivationOutput(outPath, + (outputHashRecursive ? "r:" : "") + printHashType(h.type), + h.to_string(Base16, false)); + } + + else { + /* Construct the "masked" store derivation, which is the final + one except that in the list of outputs, the output paths + are empty, and the corresponding environment variables have + an empty value. This ensures that changes in the set of + output names do get reflected in the hash. */ + for (auto & i : outputs) { + if (!jsonObject) drv.env[i] = ""; + drv.outputs[i] = DerivationOutput("", "", ""); + } + + /* Use the masked derivation expression to compute the output + path. */ + Hash h = hashDerivationModulo(*state.store, drv); + + for (auto & i : drv.outputs) + if (i.second.path == "") { + Path outPath = state.store->makeOutputPath(i.first, h, drvName); + if (!jsonObject) drv.env[i.first] = outPath; + i.second.path = outPath; + } + } + + /* Write the resulting term into the Nix store directory. */ + Path drvPath = writeDerivation(state.store, drv, drvName, state.repair); + + printMsg(lvlChatty, format("instantiated '%1%' -> '%2%'") + % drvName % drvPath); + + /* Optimisation, but required in read-only mode! because in that + case we don't actually write store derivations, so we can't + read them later. */ + drvHashes[drvPath] = hashDerivationModulo(*state.store, drv); + + state.mkAttrs(v, 1 + drv.outputs.size()); + mkString(*state.allocAttr(v, state.sDrvPath), drvPath, {"=" + drvPath}); + for (auto & i : drv.outputs) { + mkString(*state.allocAttr(v, state.symbols.create(i.first)), + i.second.path, {"!" + i.first + "!" + drvPath}); + } + v.attrs->sort(); +} + + +/* Return a placeholder string for the specified output that will be + substituted by the corresponding output path at build time. For + example, 'placeholder "out"' returns the string + /1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9. At build + time, any occurence of this string in an derivation attribute will + be replaced with the concrete path in the Nix store of the output + ‘out’. */ +static void prim_placeholder(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkString(v, hashPlaceholder(state.forceStringNoCtx(*args[0], pos))); +} + + +/************************************************************* + * Paths + *************************************************************/ + + +/* Convert the argument to a path. !!! obsolete? */ +static void prim_toPath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + mkString(v, canonPath(path), context); +} + + +/* Allow a valid store path to be used in an expression. This is + useful in some generated expressions such as in nix-push, which + generates a call to a function with an already existing store path + as argument. You don't want to use `toPath' here because it copies + the path to the Nix store, which yields a copy like + /nix/store/newhash-oldhash-oldname. In the past, `toPath' had + special case behaviour for store paths, but that created weird + corner cases. */ +static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context)); + /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink + directly in the store. The latter condition is necessary so + e.g. nix-push does the right thing. */ + if (!state.store->isStorePath(path)) path = canonPath(path, true); + if (!state.store->isInStore(path)) + throw EvalError(format("path '%1%' is not in the Nix store, at %2%") % path % pos); + Path path2 = state.store->toStorePath(path); + if (!settings.readOnlyMode) + state.store->ensurePath(path2); + context.insert(path2); + mkString(v, path, context); +} + + +static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format( + "cannot check the existence of '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + + try { + mkBool(v, pathExists(state.checkSourcePath(path))); + } catch (SysError & e) { + /* Don't give away info from errors while canonicalising + ‘path’ in restricted mode. */ + mkBool(v, false); + } catch (RestrictedPathError & e) { + mkBool(v, false); + } +} + + +/* Return the base name of the given string, i.e., everything + following the last slash. */ +static void prim_baseNameOf(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + mkString(v, baseNameOf(state.coerceToString(pos, *args[0], context, false, false)), context); +} + + +/* Return the directory of the given path, i.e., everything before the + last slash. Return either a path or a string depending on the type + of the argument. */ +static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); + if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); +} + + +/* Return the contents of a file as a string. */ +static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[0], context); + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); + if (s.find((char) 0) != string::npos) + throw Error(format("the contents of the file '%1%' cannot be represented as a Nix string") % path); + mkString(v, s.c_str()); +} + + +/* Find a file in the Nix search path. Used to implement paths, + which are desugared to 'findFile __nixPath "x"'. */ +static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + + SearchPath searchPath; + + for (unsigned int n = 0; n < args[0]->listSize(); ++n) { + Value & v2(*args[0]->listElems()[n]); + state.forceAttrs(v2, pos); + + string prefix; + Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); + if (i != v2.attrs->end()) + prefix = state.forceStringNoCtx(*i->value, pos); + + i = v2.attrs->find(state.symbols.create("path")); + if (i == v2.attrs->end()) + throw EvalError(format("attribute 'path' missing, at %1%") % pos); + + PathSet context; + string path = state.coerceToString(pos, *i->value, context, false, false); + + try { + state.realiseContext(context); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot find '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + + searchPath.emplace_back(prefix, path); + } + + string path = state.forceStringNoCtx(*args[1], pos); + + mkPath(v, state.checkSourcePath(state.findFile(searchPath, path, pos)).c_str()); +} + +/* Return the cryptographic hash of a file in base-16. */ +static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + + PathSet context; // discarded + Path p = state.coerceToPath(pos, *args[1], context); + + mkString(v, hashFile(ht, state.checkSourcePath(p)).to_string(Base16, false), context); +} + +/* Read a directory (without . or ..) */ +static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet ctx; + Path path = state.coerceToPath(pos, *args[0], ctx); + try { + state.realiseContext(ctx); + } catch (InvalidPathError & e) { + throw EvalError(format("cannot read '%1%', since path '%2%' is not valid, at %3%") + % path % e.path % pos); + } + + DirEntries entries = readDirectory(state.checkSourcePath(path)); + state.mkAttrs(v, entries.size()); + + for (auto & ent : entries) { + Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); + if (ent.type == DT_UNKNOWN) + ent.type = getFileType(path + "/" + ent.name); + mkStringNoCopy(*ent_val, + ent.type == DT_REG ? "regular" : + ent.type == DT_DIR ? "directory" : + ent.type == DT_LNK ? "symlink" : + "unknown"); + } + + v.attrs->sort(); +} + + +/************************************************************* + * Creating files + *************************************************************/ + + +/* Convert the argument (which can be any Nix expression) to an XML + representation returned in a string. Not all Nix expressions can + be sensibly or completely represented (e.g., functions). */ +static void prim_toXML(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::ostringstream out; + PathSet context; + printValueAsXML(state, true, false, *args[0], out, context); + mkString(v, out.str(), context); +} + + +/* Convert the argument (which can be any Nix expression) to a JSON + string. Not all Nix expressions can be sensibly or completely + represented (e.g., functions). */ +static void prim_toJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::ostringstream out; + PathSet context; + printValueAsJSON(state, true, *args[0], out, context); + mkString(v, out.str(), context); +} + + +/* Parse a JSON string to a value. */ +static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string s = state.forceStringNoCtx(*args[0], pos); + parseJSON(state, s, v); +} + + +/* Store a string in the Nix store as a source file that can be used + as an input by derivations. */ +static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string name = state.forceStringNoCtx(*args[0], pos); + string contents = state.forceString(*args[1], context, pos); + + PathSet refs; + + for (auto path : context) { + if (path.at(0) != '/') + throw EvalError(format("in 'toFile': the file '%1%' cannot refer to derivation outputs, at %2%") % name % pos); + refs.insert(path); + } + + Path storePath = settings.readOnlyMode + ? state.store->computeStorePathForText(name, contents, refs) + : state.store->addTextToStore(name, contents, refs, state.repair); + + /* Note: we don't need to add `context' to the context of the + result, since `storePath' itself has references to the paths + used in args[1]. */ + + mkString(v, storePath, {storePath}); +} + + +static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_, + Value * filterFun, bool recursive, const Hash & expectedHash, Value & v) +{ + const auto path = evalSettings.pureEval && expectedHash ? + path_ : + state.checkSourcePath(path_); + PathFilter filter = filterFun ? ([&](const Path & path) { + auto st = lstat(path); + + /* Call the filter function. The first argument is the path, + the second is a string indicating the type of the file. */ + Value arg1; + mkString(arg1, path); + + Value fun2; + state.callFunction(*filterFun, arg1, fun2, noPos); + + Value arg2; + mkString(arg2, + S_ISREG(st.st_mode) ? "regular" : + S_ISDIR(st.st_mode) ? "directory" : + S_ISLNK(st.st_mode) ? "symlink" : + "unknown" /* not supported, will fail! */); + + Value res; + state.callFunction(fun2, arg2, res, noPos); + + return state.forceBool(res, pos); + }) : defaultPathFilter; + + Path expectedStorePath; + if (expectedHash) { + expectedStorePath = + state.store->makeFixedOutputPath(recursive, expectedHash, name); + } + Path dstPath; + if (!expectedHash || !state.store->isValidPath(expectedStorePath)) { + dstPath = settings.readOnlyMode + ? state.store->computeStorePathForPath(name, path, recursive, htSHA256, filter).first + : state.store->addToStore(name, path, recursive, htSHA256, filter, state.repair); + if (expectedHash && expectedStorePath != dstPath) { + throw Error(format("store path mismatch in (possibly filtered) path added from '%1%'") % path); + } + } else + dstPath = expectedStorePath; + + mkString(v, dstPath, {dstPath}); +} + + +static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + Path path = state.coerceToPath(pos, *args[1], context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % pos); + + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("first argument in call to 'filterSource' is not a function but %1%, at %2%") % showType(*args[0]) % pos); + + addPath(state, pos, baseNameOf(path), path, args[0], true, Hash(), v); +} + +static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + Path path; + string name; + Value * filterFun = nullptr; + auto recursive = true; + Hash expectedHash; + + for (auto & attr : *args[0]->attrs) { + const string & n(attr.name); + if (n == "path") { + PathSet context; + path = state.coerceToPath(*attr.pos, *attr.value, context); + if (!context.empty()) + throw EvalError(format("string '%1%' cannot refer to other paths, at %2%") % path % *attr.pos); + } else if (attr.name == state.sName) + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "filter") { + state.forceValue(*attr.value); + filterFun = attr.value; + } else if (n == "recursive") + recursive = state.forceBool(*attr.value, *attr.pos); + else if (n == "sha256") + expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else + throw EvalError(format("unsupported argument '%1%' to 'addPath', at %2%") % attr.name % *attr.pos); + } + if (path.empty()) + throw EvalError(format("'path' required, at %1%") % pos); + if (name.empty()) + name = baseNameOf(path); + + addPath(state, pos, name, path, filterFun, recursive, expectedHash, v); +} + + +/************************************************************* + * Sets + *************************************************************/ + + +/* Return the names of the attributes in a set as a sorted list of + strings. */ +static void prim_attrNames(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + + state.mkList(v, args[0]->attrs->size()); + + size_t n = 0; + for (auto & i : *args[0]->attrs) + mkString(*(v.listElems()[n++] = state.allocValue()), i.name); + + std::sort(v.listElems(), v.listElems() + n, + [](Value * v1, Value * v2) { return strcmp(v1->string.s, v2->string.s) < 0; }); +} + + +/* Return the values of the attributes in a set as a list, in the same + order as attrNames. */ +static void prim_attrValues(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + + state.mkList(v, args[0]->attrs->size()); + + unsigned int n = 0; + for (auto & i : *args[0]->attrs) + v.listElems()[n++] = (Value *) &i; + + std::sort(v.listElems(), v.listElems() + n, + [](Value * v1, Value * v2) { return (string) ((Attr *) v1)->name < (string) ((Attr *) v2)->name; }); + + for (unsigned int i = 0; i < n; ++i) + v.listElems()[i] = ((Attr *) v.listElems()[i])->value; +} + + +/* Dynamic version of the `.' operator. */ +void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + // !!! Should we create a symbol here or just do a lookup? + Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); + if (i == args[1]->attrs->end()) + throw EvalError(format("attribute '%1%' missing, at %2%") % attr % pos); + // !!! add to stack trace? + if (state.countCalls && i->pos) state.attrSelects[*i->pos]++; + state.forceValue(*i->value); + v = *i->value; +} + + +/* Return position information of the specified attribute. */ +void prim_unsafeGetAttrPos(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr)); + if (i == args[1]->attrs->end()) + mkNull(v); + else + state.mkPos(v, i->pos); +} + + +/* Dynamic version of the `?' operator. */ +static void prim_hasAttr(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string attr = state.forceStringNoCtx(*args[0], pos); + state.forceAttrs(*args[1], pos); + mkBool(v, args[1]->attrs->find(state.symbols.create(attr)) != args[1]->attrs->end()); +} + + +/* Determine whether the argument is a set. */ +static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->type == tAttrs); +} + + +static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + state.forceList(*args[1], pos); + + /* Get the attribute names to be removed. */ + std::set names; + for (unsigned int i = 0; i < args[1]->listSize(); ++i) { + state.forceStringNoCtx(*args[1]->listElems()[i], pos); + names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); + } + + /* Copy all attributes not in that set. Note that we don't need + to sort v.attrs because it's a subset of an already sorted + vector. */ + state.mkAttrs(v, args[0]->attrs->size()); + for (auto & i : *args[0]->attrs) { + if (names.find(i.name) == names.end()) + v.attrs->push_back(i); + } +} + + +/* Builds a set from a list specifying (name, value) pairs. To be + precise, a list [{name = "name1"; value = value1;} ... {name = + "nameN"; value = valueN;}] is transformed to {name1 = value1; + ... nameN = valueN;}. In case of duplicate occurences of the same + name, the first takes precedence. */ +static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + + state.mkAttrs(v, args[0]->listSize()); + + std::set seen; + + for (unsigned int i = 0; i < args[0]->listSize(); ++i) { + Value & v2(*args[0]->listElems()[i]); + state.forceAttrs(v2, pos); + + Bindings::iterator j = v2.attrs->find(state.sName); + if (j == v2.attrs->end()) + throw TypeError(format("'name' attribute missing in a call to 'listToAttrs', at %1%") % pos); + string name = state.forceStringNoCtx(*j->value, pos); + + Symbol sym = state.symbols.create(name); + if (seen.find(sym) == seen.end()) { + Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue)); + if (j2 == v2.attrs->end()) + throw TypeError(format("'value' attribute missing in a call to 'listToAttrs', at %1%") % pos); + + v.attrs->push_back(Attr(sym, j2->value, j2->pos)); + seen.insert(sym); + } + } + + v.attrs->sort(); +} + + +/* Return the right-biased intersection of two sets as1 and as2, + i.e. a set that contains every attribute from as2 that is also a + member of as1. */ +static void prim_intersectAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[0], pos); + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, std::min(args[0]->attrs->size(), args[1]->attrs->size())); + + for (auto & i : *args[0]->attrs) { + Bindings::iterator j = args[1]->attrs->find(i.name); + if (j != args[1]->attrs->end()) + v.attrs->push_back(*j); + } +} + + +/* Collect each attribute named `attr' from a list of attribute sets. + Sets that don't contain the named attribute are ignored. + + Example: + catAttrs "a" [{a = 1;} {b = 0;} {a = 2;}] + => [1 2] +*/ +static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + Symbol attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos)); + state.forceList(*args[1], pos); + + Value * res[args[1]->listSize()]; + unsigned int found = 0; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value & v2(*args[1]->listElems()[n]); + state.forceAttrs(v2, pos); + Bindings::iterator i = v2.attrs->find(attrName); + if (i != v2.attrs->end()) + res[found++] = i->value; + } + + state.mkList(v, found); + for (unsigned int n = 0; n < found; ++n) + v.listElems()[n] = res[n]; +} + + +/* Return a set containing the names of the formal arguments expected + by the function `f'. The value of each attribute is a Boolean + denoting whether the corresponding argument has a default value. For instance, + + functionArgs ({ x, y ? 123}: ...) + => { x = false; y = true; } + + "Formal argument" here refers to the attributes pattern-matched by + the function. Plain lambdas are not included, e.g. + + functionArgs (x: ...) + => { } +*/ +static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + if (args[0]->type != tLambda) + throw TypeError(format("'functionArgs' requires a function, at %1%") % pos); + + if (!args[0]->lambda.fun->matchAttrs) { + state.mkAttrs(v, 0); + return; + } + + state.mkAttrs(v, args[0]->lambda.fun->formals->formals.size()); + for (auto & i : args[0]->lambda.fun->formals->formals) + // !!! should optimise booleans (allocate only once) + mkBool(*state.allocAttr(v, i.name), i.def); + v.attrs->sort(); +} + + +/* Apply a function to every element of an attribute set. */ +static void prim_mapAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceAttrs(*args[1], pos); + + state.mkAttrs(v, args[1]->attrs->size()); + + for (auto & i : *args[1]->attrs) { + Value * vName = state.allocValue(); + Value * vFun2 = state.allocValue(); + mkString(*vName, i.name); + mkApp(*vFun2, *args[0], *vName); + mkApp(*state.allocAttr(v, i.name), *vFun2, *i.value); + } +} + + + +/************************************************************* + * Lists + *************************************************************/ + + +/* Determine whether the argument is a list. */ +static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + mkBool(v, args[0]->isList()); +} + + +static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Value & v) +{ + state.forceList(list, pos); + if (n < 0 || (unsigned int) n >= list.listSize()) + throw Error(format("list index %1% is out of bounds, at %2%") % n % pos); + state.forceValue(*list.listElems()[n]); + v = *list.listElems()[n]; +} + + +/* Return the n-1'th element of a list. */ +static void prim_elemAt(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + elemAt(state, pos, *args[0], state.forceInt(*args[1], pos), v); +} + + +/* Return the first element of a list. */ +static void prim_head(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + elemAt(state, pos, *args[0], 0, v); +} + + +/* Return a list consisting of everything but the first element of + a list. Warning: this function takes O(n) time, so you probably + don't want to use it! */ +static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + if (args[0]->listSize() == 0) + throw Error(format("'tail' called on an empty list, at %1%") % pos); + state.mkList(v, args[0]->listSize() - 1); + for (unsigned int n = 0; n < v.listSize(); ++n) + v.listElems()[n] = args[0]->listElems()[n + 1]; +} + + +/* Apply a function to every element of a list. */ +static void prim_map(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[1], pos); + + state.mkList(v, args[1]->listSize()); + + for (unsigned int n = 0; n < v.listSize(); ++n) + mkApp(*(v.listElems()[n] = state.allocValue()), + *args[0], *args[1]->listElems()[n]); +} + + +/* Filter a list using a predicate; that is, return a list containing + every element from the list for which the predicate function + returns true. */ +static void prim_filter(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + // FIXME: putting this on the stack is risky. + Value * vs[args[1]->listSize()]; + unsigned int k = 0; + + bool same = true; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + Value res; + state.callFunction(*args[0], *args[1]->listElems()[n], res, noPos); + if (state.forceBool(res, pos)) + vs[k++] = args[1]->listElems()[n]; + else + same = false; + } + + if (same) + v = *args[1]; + else { + state.mkList(v, k); + for (unsigned int n = 0; n < k; ++n) v.listElems()[n] = vs[n]; + } +} + + +/* Return true if a list contains a given element. */ +static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + bool res = false; + state.forceList(*args[1], pos); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) + if (state.eqValues(*args[0], *args[1]->listElems()[n])) { + res = true; + break; + } + mkBool(v, res); +} + + +/* Concatenate a list of lists. */ +static void prim_concatLists(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + state.concatLists(v, args[0]->listSize(), args[0]->listElems(), pos); +} + + +/* Return the length of a list. This is an O(1) time operation. */ +static void prim_length(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + mkInt(v, args[0]->listSize()); +} + + +/* Reduce a list by applying a binary operator, from left to + right. The operator is applied strictly. */ +static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[2], pos); + + if (args[2]->listSize()) { + Value * vCur = args[1]; + + for (unsigned int n = 0; n < args[2]->listSize(); ++n) { + Value vTmp; + state.callFunction(*args[0], *vCur, vTmp, pos); + vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); + state.callFunction(vTmp, *args[2]->listElems()[n], *vCur, pos); + } + state.forceValue(v); + } else { + state.forceValue(*args[1]); + v = *args[1]; + } +} + + +static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + Value vTmp; + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + bool res = state.forceBool(vTmp, pos); + if (res == any) { + mkBool(v, any); + return; + } + } + + mkBool(v, !any); +} + + +static void prim_any(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + anyOrAll(true, state, pos, args, v); +} + + +static void prim_all(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + anyOrAll(false, state, pos, args, v); +} + + +static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto len = state.forceInt(*args[1], pos); + + if (len < 0) + throw EvalError(format("cannot create list of size %1%, at %2%") % len % pos); + + state.mkList(v, len); + + for (unsigned int n = 0; n < (unsigned int) len; ++n) { + Value * arg = state.allocValue(); + mkInt(*arg, n); + mkApp(*(v.listElems()[n] = state.allocValue()), *args[0], *arg); + } +} + + +static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v); + + +static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + auto len = args[1]->listSize(); + state.mkList(v, len); + for (unsigned int n = 0; n < len; ++n) { + state.forceValue(*args[1]->listElems()[n]); + v.listElems()[n] = args[1]->listElems()[n]; + } + + + auto comparator = [&](Value * a, Value * b) { + /* Optimization: if the comparator is lessThan, bypass + callFunction. */ + if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + return CompareValues()(a, b); + + Value vTmp1, vTmp2; + state.callFunction(*args[0], *a, vTmp1, pos); + state.callFunction(vTmp1, *b, vTmp2, pos); + return state.forceBool(vTmp2, pos); + }; + + /* FIXME: std::sort can segfault if the comparator is not a strict + weak ordering. What to do? std::stable_sort() seems more + resilient, but no guarantees... */ + std::stable_sort(v.listElems(), v.listElems() + len, comparator); +} + + +static void prim_partition(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + auto len = args[1]->listSize(); + + ValueVector right, wrong; + + for (unsigned int n = 0; n < len; ++n) { + auto vElem = args[1]->listElems()[n]; + state.forceValue(*vElem); + Value res; + state.callFunction(*args[0], *vElem, res, pos); + if (state.forceBool(res, pos)) + right.push_back(vElem); + else + wrong.push_back(vElem); + } + + state.mkAttrs(v, 2); + + Value * vRight = state.allocAttr(v, state.sRight); + auto rsize = right.size(); + state.mkList(*vRight, rsize); + if (rsize) + memcpy(vRight->listElems(), right.data(), sizeof(Value *) * rsize); + + Value * vWrong = state.allocAttr(v, state.sWrong); + auto wsize = wrong.size(); + state.mkList(*vWrong, wsize); + if (wsize) + memcpy(vWrong->listElems(), wrong.data(), sizeof(Value *) * wsize); + + v.attrs->sort(); +} + + +/* concatMap = f: list: concatLists (map f list); */ +/* C++-version is to avoid allocating `mkApp', call `f' eagerly */ +static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + auto nrLists = args[1]->listSize(); + + Value lists[nrLists]; + size_t len = 0; + + for (unsigned int n = 0; n < nrLists; ++n) { + Value * vElem = args[1]->listElems()[n]; + state.callFunction(*args[0], *vElem, lists[n], pos); + state.forceList(lists[n], pos); + len += lists[n].listSize(); + } + + state.mkList(v, len); + auto out = v.listElems(); + for (unsigned int n = 0, pos = 0; n < nrLists; ++n) { + auto l = lists[n].listSize(); + if (l) + memcpy(out + pos, lists[n].listElems(), l * sizeof(Value *)); + pos += l; + } +} + + +/************************************************************* + * Integer arithmetic + *************************************************************/ + + +static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); +} + + +static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); +} + + +static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + if (args[0]->type == tFloat || args[1]->type == tFloat) + mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); + else + mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); +} + + +static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0], pos); + state.forceValue(*args[1], pos); + + NixFloat f2 = state.forceFloat(*args[1], pos); + if (f2 == 0) throw EvalError(format("division by zero, at %1%") % pos); + + if (args[0]->type == tFloat || args[1]->type == tFloat) { + mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); + } else { + NixInt i1 = state.forceInt(*args[0], pos); + NixInt i2 = state.forceInt(*args[1], pos); + /* Avoid division overflow as it might raise SIGFPE. */ + if (i1 == std::numeric_limits::min() && i2 == -1) + throw EvalError(format("overflow in integer division, at %1%") % pos); + mkInt(v, i1 / i2); + } +} + +static void prim_bitAnd(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) & state.forceInt(*args[1], pos)); +} + +static void prim_bitOr(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) | state.forceInt(*args[1], pos)); +} + +static void prim_bitXor(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + mkInt(v, state.forceInt(*args[0], pos) ^ state.forceInt(*args[1], pos)); +} + +static void prim_lessThan(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceValue(*args[0]); + state.forceValue(*args[1]); + CompareValues comp; + mkBool(v, comp(args[0], args[1])); +} + + +/************************************************************* + * String manipulation + *************************************************************/ + + +/* Convert the argument to a string. Paths are *not* copied to the + store, so `toString /foo/bar' yields `"/foo/bar"', not + `"/nix/store/whatever..."'. */ +static void prim_toString(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context, true, false); + mkString(v, s, context); +} + + +/* `substring start len str' returns the substring of `str' starting + at character position `min(start, stringLength str)' inclusive and + ending at `min(start + len, stringLength str)'. `start' must be + non-negative. */ +static void prim_substring(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + int start = state.forceInt(*args[0], pos); + int len = state.forceInt(*args[1], pos); + PathSet context; + string s = state.coerceToString(pos, *args[2], context); + + if (start < 0) throw EvalError(format("negative start position in 'substring', at %1%") % pos); + + mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context); +} + + +static void prim_stringLength(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkInt(v, s.size()); +} + + +/* Return the cryptographic hash of a string in base-16. */ +static void prim_hashString(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string type = state.forceStringNoCtx(*args[0], pos); + HashType ht = parseHashType(type); + if (ht == htUnknown) + throw Error(format("unknown hash type '%1%', at %2%") % type % pos); + + PathSet context; // discarded + string s = state.forceString(*args[1], context, pos); + + mkString(v, hashString(ht, s).to_string(Base16, false), context); +} + + +/* Match a regular expression against a string and return either + ‘null’ or a list containing substring matches. */ +static void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + std::smatch match; + if (!std::regex_match(str, match, regex)) { + mkNull(v); + return; + } + + // the first match is the whole string + const size_t len = match.size() - 1; + state.mkList(v, len); + for (size_t i = 0; i < len; ++i) { + if (!match[i+1].matched) + mkNull(*(v.listElems()[i] = state.allocValue())); + else + mkString(*(v.listElems()[i] = state.allocValue()), match[i + 1].str().c_str()); + } + + } catch (std::regex_error &e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); + } else { + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } + } +} + + +/* Split a string with a regular expression, and return a list of the + non-matching parts interleaved by the lists of the matching groups. */ +static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto re = state.forceStringNoCtx(*args[0], pos); + + try { + + std::regex regex(re, std::regex::extended); + + PathSet context; + const std::string str = state.forceString(*args[1], context, pos); + + auto begin = std::sregex_iterator(str.begin(), str.end(), regex); + auto end = std::sregex_iterator(); + + // Any matches results are surrounded by non-matching results. + const size_t len = std::distance(begin, end); + state.mkList(v, 2 * len + 1); + size_t idx = 0; + Value * elem; + + if (len == 0) { + v.listElems()[idx++] = args[1]; + return; + } + + for (std::sregex_iterator i = begin; i != end; ++i) { + assert(idx <= 2 * len + 1 - 3); + std::smatch match = *i; + + // Add a string for non-matched characters. + elem = v.listElems()[idx++] = state.allocValue(); + mkString(*elem, match.prefix().str().c_str()); + + // Add a list for matched substrings. + const size_t slen = match.size() - 1; + elem = v.listElems()[idx++] = state.allocValue(); + + // Start at 1, beacause the first match is the whole string. + state.mkList(*elem, slen); + for (size_t si = 0; si < slen; ++si) { + if (!match[si + 1].matched) + mkNull(*(elem->listElems()[si] = state.allocValue())); + else + mkString(*(elem->listElems()[si] = state.allocValue()), match[si + 1].str().c_str()); + } + + // Add a string for non-matched suffix characters. + if (idx == 2 * len) { + elem = v.listElems()[idx++] = state.allocValue(); + mkString(*elem, match.suffix().str().c_str()); + } + } + assert(idx == 2 * len + 1); + + } catch (std::regex_error &e) { + if (e.code() == std::regex_constants::error_space) { + // limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++ + throw EvalError("memory limit exceeded by regular expression '%s', at %s", re, pos); + } else { + throw EvalError("invalid regular expression '%s', at %s", re, pos); + } + } +} + + +static void prim_concatStringSep(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + + auto sep = state.forceString(*args[0], context, pos); + state.forceList(*args[1], pos); + + string res; + res.reserve((args[1]->listSize() + 32) * sep.size()); + bool first = true; + + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + if (first) first = false; else res += sep; + res += state.coerceToString(pos, *args[1]->listElems()[n], context); + } + + mkString(v, res, context); +} + + +static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceList(*args[0], pos); + state.forceList(*args[1], pos); + if (args[0]->listSize() != args[1]->listSize()) + throw EvalError(format("'from' and 'to' arguments to 'replaceStrings' have different lengths, at %1%") % pos); + + vector from; + from.reserve(args[0]->listSize()); + for (unsigned int n = 0; n < args[0]->listSize(); ++n) + from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + + vector> to; + to.reserve(args[1]->listSize()); + for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + PathSet ctx; + auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); + to.push_back(std::make_pair(std::move(s), std::move(ctx))); + } + + PathSet context; + auto s = state.forceString(*args[2], context, pos); + + string res; + // Loops one past last character to handle the case where 'from' contains an empty string. + for (size_t p = 0; p <= s.size(); ) { + bool found = false; + auto i = from.begin(); + auto j = to.begin(); + for (; i != from.end(); ++i, ++j) + if (s.compare(p, i->size(), *i) == 0) { + found = true; + res += j->first; + if (i->empty()) { + if (p < s.size()) + res += s[p]; + p++; + } else { + p += i->size(); + } + for (auto& path : j->second) + context.insert(path); + j->second.clear(); + break; + } + if (!found) { + if (p < s.size()) + res += s[p]; + p++; + } + } + + mkString(v, res, context); +} + + +/************************************************************* + * Versions + *************************************************************/ + + +static void prim_parseDrvName(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string name = state.forceStringNoCtx(*args[0], pos); + DrvName parsed(name); + state.mkAttrs(v, 2); + mkString(*state.allocAttr(v, state.sName), parsed.name); + mkString(*state.allocAttr(v, state.symbols.create("version")), parsed.version); + v.attrs->sort(); +} + + +static void prim_compareVersions(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string version1 = state.forceStringNoCtx(*args[0], pos); + string version2 = state.forceStringNoCtx(*args[1], pos); + mkInt(v, compareVersions(version1, version2)); +} + + +static void prim_splitVersion(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + string version = state.forceStringNoCtx(*args[0], pos); + auto iter = version.cbegin(); + Strings components; + while (iter != version.cend()) { + auto component = nextComponent(iter, version.cend()); + if (component.empty()) + break; + components.emplace_back(std::move(component)); + } + state.mkList(v, components.size()); + unsigned int n = 0; + for (auto & component : components) { + auto listElem = v.listElems()[n++] = state.allocValue(); + mkString(*listElem, std::move(component)); + } +} + + +/************************************************************* + * Networking + *************************************************************/ + + +void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, + const string & who, bool unpack, const std::string & defaultName) +{ + CachedDownloadRequest request(""); + request.unpack = unpack; + request.name = defaultName; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + request.uri = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "sha256") + request.expectedHash = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256); + else if (n == "name") + request.name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError(format("unsupported argument '%1%' to '%2%', at %3%") % attr.name % who % attr.pos); + } + + if (request.uri.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); + + } else + request.uri = state.forceStringNoCtx(*args[0], pos); + + state.checkURI(request.uri); + + if (evalSettings.pureEval && !request.expectedHash) + throw Error("in pure evaluation mode, '%s' requires a 'sha256' argument", who); + + auto res = getDownloader()->downloadCached(state.store, request); + + if (state.allowedPaths) + state.allowedPaths->insert(res.path); + + mkString(v, res.storePath, PathSet({res.storePath})); +} + + +static void prim_fetchurl(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchurl", false, ""); +} + + +static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + fetch(state, pos, args, v, "fetchTarball", true, "source"); +} + + +/************************************************************* + * Primop registration + *************************************************************/ + + +RegisterPrimOp::PrimOps * RegisterPrimOp::primOps; + + +RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun) +{ + if (!primOps) primOps = new PrimOps; + primOps->emplace_back(name, arity, fun); +} + + +void EvalState::createBaseEnv() +{ + baseEnv.up = 0; + + /* Add global constants such as `true' to the base environment. */ + Value v; + + /* `builtins' must be first! */ + mkAttrs(v, 128); + addConstant("builtins", v); + + mkBool(v, true); + addConstant("true", v); + + mkBool(v, false); + addConstant("false", v); + + mkNull(v); + addConstant("null", v); + + auto vThrow = addPrimOp("throw", 1, prim_throw); + + auto addPurityError = [&](const std::string & name) { + Value * v2 = allocValue(); + mkString(*v2, fmt("'%s' is not allowed in pure evaluation mode", name)); + mkApp(v, *vThrow, *v2); + addConstant(name, v); + }; + + if (!evalSettings.pureEval) { + mkInt(v, time(0)); + addConstant("__currentTime", v); + } + + if (!evalSettings.pureEval) { + mkString(v, settings.thisSystem); + addConstant("__currentSystem", v); + } + + mkString(v, nixVersion); + addConstant("__nixVersion", v); + + mkString(v, store->storeDir); + addConstant("__storeDir", v); + + /* Language version. This should be increased every time a new + language feature gets added. It's not necessary to increase it + when primops get added, because you can just use `builtins ? + primOp' to check. */ + mkInt(v, 5); + addConstant("__langVersion", v); + + // Miscellaneous + auto vScopedImport = addPrimOp("scopedImport", 2, prim_scopedImport); + Value * v2 = allocValue(); + mkAttrs(*v2, 0); + mkApp(v, *vScopedImport, *v2); + forceValue(v); + addConstant("import", v); + if (evalSettings.enableNativeCode) { + addPrimOp("__importNative", 2, prim_importNative); + addPrimOp("__exec", 1, prim_exec); + } + addPrimOp("__typeOf", 1, prim_typeOf); + addPrimOp("isNull", 1, prim_isNull); + addPrimOp("__isFunction", 1, prim_isFunction); + addPrimOp("__isString", 1, prim_isString); + addPrimOp("__isInt", 1, prim_isInt); + addPrimOp("__isFloat", 1, prim_isFloat); + addPrimOp("__isBool", 1, prim_isBool); + addPrimOp("__isPath", 1, prim_isPath); + addPrimOp("__genericClosure", 1, prim_genericClosure); + addPrimOp("abort", 1, prim_abort); + addPrimOp("__addErrorContext", 2, prim_addErrorContext); + addPrimOp("__tryEval", 1, prim_tryEval); + addPrimOp("__getEnv", 1, prim_getEnv); + + // Strictness + addPrimOp("__seq", 2, prim_seq); + addPrimOp("__deepSeq", 2, prim_deepSeq); + + // Debugging + addPrimOp("__trace", 2, prim_trace); + addPrimOp("__valueSize", 1, prim_valueSize); + + // Paths + addPrimOp("__toPath", 1, prim_toPath); + if (evalSettings.pureEval) + addPurityError("__storePath"); + else + addPrimOp("__storePath", 1, prim_storePath); + addPrimOp("__pathExists", 1, prim_pathExists); + addPrimOp("baseNameOf", 1, prim_baseNameOf); + addPrimOp("dirOf", 1, prim_dirOf); + addPrimOp("__readFile", 1, prim_readFile); + addPrimOp("__readDir", 1, prim_readDir); + addPrimOp("__findFile", 2, prim_findFile); + addPrimOp("__hashFile", 2, prim_hashFile); + + // Creating files + addPrimOp("__toXML", 1, prim_toXML); + addPrimOp("__toJSON", 1, prim_toJSON); + addPrimOp("__fromJSON", 1, prim_fromJSON); + addPrimOp("__toFile", 2, prim_toFile); + addPrimOp("__filterSource", 2, prim_filterSource); + addPrimOp("__path", 1, prim_path); + + // Sets + addPrimOp("__attrNames", 1, prim_attrNames); + addPrimOp("__attrValues", 1, prim_attrValues); + addPrimOp("__getAttr", 2, prim_getAttr); + addPrimOp("__unsafeGetAttrPos", 2, prim_unsafeGetAttrPos); + addPrimOp("__hasAttr", 2, prim_hasAttr); + addPrimOp("__isAttrs", 1, prim_isAttrs); + addPrimOp("removeAttrs", 2, prim_removeAttrs); + addPrimOp("__listToAttrs", 1, prim_listToAttrs); + addPrimOp("__intersectAttrs", 2, prim_intersectAttrs); + addPrimOp("__catAttrs", 2, prim_catAttrs); + addPrimOp("__functionArgs", 1, prim_functionArgs); + addPrimOp("__mapAttrs", 2, prim_mapAttrs); + + // Lists + addPrimOp("__isList", 1, prim_isList); + addPrimOp("__elemAt", 2, prim_elemAt); + addPrimOp("__head", 1, prim_head); + addPrimOp("__tail", 1, prim_tail); + addPrimOp("map", 2, prim_map); + addPrimOp("__filter", 2, prim_filter); + addPrimOp("__elem", 2, prim_elem); + addPrimOp("__concatLists", 1, prim_concatLists); + addPrimOp("__length", 1, prim_length); + addPrimOp("__foldl'", 3, prim_foldlStrict); + addPrimOp("__any", 2, prim_any); + addPrimOp("__all", 2, prim_all); + addPrimOp("__genList", 2, prim_genList); + addPrimOp("__sort", 2, prim_sort); + addPrimOp("__partition", 2, prim_partition); + addPrimOp("__concatMap", 2, prim_concatMap); + + // Integer arithmetic + addPrimOp("__add", 2, prim_add); + addPrimOp("__sub", 2, prim_sub); + addPrimOp("__mul", 2, prim_mul); + addPrimOp("__div", 2, prim_div); + addPrimOp("__bitAnd", 2, prim_bitAnd); + addPrimOp("__bitOr", 2, prim_bitOr); + addPrimOp("__bitXor", 2, prim_bitXor); + addPrimOp("__lessThan", 2, prim_lessThan); + + // String manipulation + addPrimOp("toString", 1, prim_toString); + addPrimOp("__substring", 3, prim_substring); + addPrimOp("__stringLength", 1, prim_stringLength); + addPrimOp("__hashString", 2, prim_hashString); + addPrimOp("__match", 2, prim_match); + addPrimOp("__split", 2, prim_split); + addPrimOp("__concatStringsSep", 2, prim_concatStringSep); + addPrimOp("__replaceStrings", 3, prim_replaceStrings); + + // Versions + addPrimOp("__parseDrvName", 1, prim_parseDrvName); + addPrimOp("__compareVersions", 2, prim_compareVersions); + addPrimOp("__splitVersion", 1, prim_splitVersion); + + // Derivations + addPrimOp("derivationStrict", 1, prim_derivationStrict); + addPrimOp("placeholder", 1, prim_placeholder); + + // Networking + addPrimOp("__fetchurl", 1, prim_fetchurl); + addPrimOp("fetchTarball", 1, prim_fetchTarball); + + /* Add a wrapper around the derivation primop that computes the + `drvPath' and `outPath' attributes lazily. */ + string path = canonPath(settings.nixDataDir + "/nix/corepkgs/derivation.nix", true); + sDerivationNix = symbols.create(path); + evalFile(path, v); + addConstant("derivation", v); + + /* Add a value containing the current Nix expression search path. */ + mkList(v, searchPath.size()); + int n = 0; + for (auto & i : searchPath) { + v2 = v.listElems()[n++] = allocValue(); + mkAttrs(*v2, 2); + mkString(*allocAttr(*v2, symbols.create("path")), i.second); + mkString(*allocAttr(*v2, symbols.create("prefix")), i.first); + v2->attrs->sort(); + } + addConstant("__nixPath", v); + + if (RegisterPrimOp::primOps) + for (auto & primOp : *RegisterPrimOp::primOps) + addPrimOp(std::get<0>(primOp), std::get<1>(primOp), std::get<2>(primOp)); + + /* Now that we've added all primops, sort the `builtins' set, + because attribute lookups expect it to be sorted. */ + baseEnv.values[0]->attrs->sort(); +} + + +} diff --git a/third_party/nix/src/libexpr/primops.hh b/third_party/nix/src/libexpr/primops.hh new file mode 100644 index 000000000..c790b30f6 --- /dev/null +++ b/third_party/nix/src/libexpr/primops.hh @@ -0,0 +1,26 @@ +#include "eval.hh" + +#include +#include + +namespace nix { + +struct RegisterPrimOp +{ + typedef std::vector> PrimOps; + static PrimOps * primOps; + /* You can register a constant by passing an arity of 0. fun + will get called during EvalState initialization, so there + may be primops not yet added and builtins is not yet sorted. */ + RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun); +}; + +/* These primops are disabled without enableNativeCode, but plugins + may wish to use them in limited contexts without globally enabling + them. */ +/* Load a ValueInitializer from a DSO and return whatever it initializes */ +void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v); +/* Execute a program and parse its output */ +void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v); + +} diff --git a/third_party/nix/src/libexpr/primops/context.cc b/third_party/nix/src/libexpr/primops/context.cc new file mode 100644 index 000000000..2d79739ea --- /dev/null +++ b/third_party/nix/src/libexpr/primops/context.cc @@ -0,0 +1,187 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "derivations.hh" + +namespace nix { + +static void prim_unsafeDiscardStringContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + mkString(v, s, PathSet()); +} + +static RegisterPrimOp r1("__unsafeDiscardStringContext", 1, prim_unsafeDiscardStringContext); + + +static void prim_hasContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + state.forceString(*args[0], context, pos); + mkBool(v, !context.empty()); +} + +static RegisterPrimOp r2("__hasContext", 1, prim_hasContext); + + +/* Sometimes we want to pass a derivation path (i.e. pkg.drvPath) to a + builder without causing the derivation to be built (for instance, + in the derivation that builds NARs in nix-push, when doing + source-only deployment). This primop marks the string context so + that builtins.derivation adds the path to drv.inputSrcs rather than + drv.inputDrvs. */ +static void prim_unsafeDiscardOutputDependency(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + string s = state.coerceToString(pos, *args[0], context); + + PathSet context2; + for (auto & p : context) + context2.insert(p.at(0) == '=' ? string(p, 1) : p); + + mkString(v, s, context2); +} + +static RegisterPrimOp r3("__unsafeDiscardOutputDependency", 1, prim_unsafeDiscardOutputDependency); + + +/* Extract the context of a string as a structured Nix value. + + The context is represented as an attribute set whose keys are the + paths in the context set and whose values are attribute sets with + the following keys: + path: True if the relevant path is in the context as a plain store + path (i.e. the kind of context you get when interpolating + a Nix path (e.g. ./.) into a string). False if missing. + allOutputs: True if the relevant path is a derivation and it is + in the context as a drv file with all of its outputs + (i.e. the kind of context you get when referencing + .drvPath of some derivation). False if missing. + outputs: If a non-empty list, the relevant path is a derivation + and the provided outputs are referenced in the context + (i.e. the kind of context you get when referencing + .outPath of some derivation). Empty list if missing. + Note that for a given path any combination of the above attributes + may be present. +*/ +static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + struct ContextInfo { + bool path = false; + bool allOutputs = false; + Strings outputs; + }; + PathSet context; + state.forceString(*args[0], context, pos); + auto contextInfos = std::map(); + for (const auto & p : context) { + Path drv; + string output; + const Path * path = &p; + if (p.at(0) == '=') { + drv = string(p, 1); + path = &drv; + } else if (p.at(0) == '!') { + std::pair ctx = decodeContext(p); + drv = ctx.first; + output = ctx.second; + path = &drv; + } + auto isPath = drv.empty(); + auto isAllOutputs = (!drv.empty()) && output.empty(); + + auto iter = contextInfos.find(*path); + if (iter == contextInfos.end()) { + contextInfos.emplace(*path, ContextInfo{isPath, isAllOutputs, output.empty() ? Strings{} : Strings{std::move(output)}}); + } else { + if (isPath) + iter->second.path = true; + else if (isAllOutputs) + iter->second.allOutputs = true; + else + iter->second.outputs.emplace_back(std::move(output)); + } + } + + state.mkAttrs(v, contextInfos.size()); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (const auto & info : contextInfos) { + auto & infoVal = *state.allocAttr(v, state.symbols.create(info.first)); + state.mkAttrs(infoVal, 3); + if (info.second.path) + mkBool(*state.allocAttr(infoVal, sPath), true); + if (info.second.allOutputs) + mkBool(*state.allocAttr(infoVal, sAllOutputs), true); + if (!info.second.outputs.empty()) { + auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); + state.mkList(outputsVal, info.second.outputs.size()); + size_t i = 0; + for (const auto & output : info.second.outputs) { + mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); + } + } + infoVal.attrs->sort(); + } + v.attrs->sort(); +} + +static RegisterPrimOp r4("__getContext", 1, prim_getContext); + + +/* Append the given context to a given string. + + See the commentary above unsafeGetContext for details of the + context representation. +*/ +static void prim_appendContext(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + PathSet context; + auto orig = state.forceString(*args[0], context, pos); + + state.forceAttrs(*args[1], pos); + + auto sPath = state.symbols.create("path"); + auto sAllOutputs = state.symbols.create("allOutputs"); + for (auto & i : *args[1]->attrs) { + if (!state.store->isStorePath(i.name)) + throw EvalError("Context key '%s' is not a store path, at %s", i.name, i.pos); + if (!settings.readOnlyMode) + state.store->ensurePath(i.name); + state.forceAttrs(*i.value, *i.pos); + auto iter = i.value->attrs->find(sPath); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) + context.insert(i.name); + } + + iter = i.value->attrs->find(sAllOutputs); + if (iter != i.value->attrs->end()) { + if (state.forceBool(*iter->value, *iter->pos)) { + if (!isDerivation(i.name)) { + throw EvalError("Tried to add all-outputs context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + context.insert("=" + string(i.name)); + } + } + + iter = i.value->attrs->find(state.sOutputs); + if (iter != i.value->attrs->end()) { + state.forceList(*iter->value, *iter->pos); + if (iter->value->listSize() && !isDerivation(i.name)) { + throw EvalError("Tried to add derivation output context of %s, which is not a derivation, to a string, at %s", i.name, i.pos); + } + for (unsigned int n = 0; n < iter->value->listSize(); ++n) { + auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + context.insert("!" + name + "!" + string(i.name)); + } + } + } + + mkString(v, orig, context); +} + +static RegisterPrimOp r5("__appendContext", 2, prim_appendContext); + +} diff --git a/third_party/nix/src/libexpr/primops/fetchGit.cc b/third_party/nix/src/libexpr/primops/fetchGit.cc new file mode 100644 index 000000000..90f600284 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fetchGit.cc @@ -0,0 +1,247 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "store-api.hh" +#include "pathlocks.hh" +#include "hash.hh" + +#include + +#include + +#include + +using namespace std::string_literals; + +namespace nix { + +struct GitInfo +{ + Path storePath; + std::string rev; + std::string shortRev; + uint64_t revCount = 0; +}; + +std::regex revRegex("^[0-9a-fA-F]{40}$"); + +GitInfo exportGit(ref store, const std::string & uri, + std::optional ref, std::string rev, + const std::string & name) +{ + if (evalSettings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision"); + + if (!ref && rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.git")) { + + bool clean = true; + + try { + runProgram("git", true, { "-C", uri, "diff-index", "--quiet", "HEAD", "--" }); + } catch (ExecError & e) { + if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; + clean = false; + } + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + GitInfo gitInfo; + gitInfo.rev = "0000000000000000000000000000000000000000"; + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + auto files = tokenizeString>( + runProgram("git", true, { "-C", uri, "ls-files", "-z" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, uri)); + std::string file(p, uri.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + gitInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + + return gitInfo; + } + + // clean working tree, but no ref or rev specified. Use 'HEAD'. + rev = chomp(runProgram("git", true, { "-C", uri, "rev-parse", "HEAD" })); + ref = "HEAD"s; + } + + if (!ref) ref = "HEAD"s; + + if (rev != "" && !std::regex_match(rev, revRegex)) + throw Error("invalid Git revision '%s'", rev); + + deletePath(getCacheDir() + "/nix/git"); + + Path cacheDir = getCacheDir() + "/nix/gitv2/" + hashString(htSHA256, uri).to_string(Base32, false); + + if (!pathExists(cacheDir)) { + createDirs(dirOf(cacheDir)); + runProgram("git", true, { "init", "--bare", cacheDir }); + } + + Path localRefFile; + if (ref->compare(0, 5, "refs/") == 0) + localRefFile = cacheDir + "/" + *ref; + else + localRefFile = cacheDir + "/refs/heads/" + *ref; + + bool doFetch; + time_t now = time(0); + /* If a rev was specified, we need to fetch if it's not in the + repo. */ + if (rev != "") { + try { + runProgram("git", true, { "-C", cacheDir, "cat-file", "-e", rev }); + doFetch = false; + } catch (ExecError & e) { + if (WIFEXITED(e.status)) { + doFetch = true; + } else { + throw; + } + } + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } + if (doFetch) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", uri)); + + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + runProgram("git", true, { "-C", cacheDir, "fetch", "--quiet", "--force", "--", uri, fmt("%s:%s", *ref, *ref) }); + + struct timeval times[2]; + times[0].tv_sec = now; + times[0].tv_usec = 0; + times[1].tv_sec = now; + times[1].tv_usec = 0; + + utimes(localRefFile.c_str(), times); + } + + // FIXME: check whether rev is an ancestor of ref. + GitInfo gitInfo; + gitInfo.rev = rev != "" ? rev : chomp(readFile(localRefFile)); + gitInfo.shortRev = std::string(gitInfo.rev, 0, 7); + + printTalkative("using revision %s of repo '%s'", gitInfo.rev, uri); + + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + gitInfo.rev).to_string(Base32, false); + Path storeLink = cacheDir + "/" + storeLinkName + ".link"; + PathLocks storeLinkLock({storeLink}, fmt("waiting for lock on '%1%'...", storeLink)); // FIXME: broken + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == gitInfo.rev); + + gitInfo.storePath = json["storePath"]; + + if (store->isValidPath(gitInfo.storePath)) { + gitInfo.revCount = json["revCount"]; + return gitInfo; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + // FIXME: should pipe this, or find some better way to extract a + // revision. + auto tar = runProgram("git", true, { "-C", cacheDir, "archive", gitInfo.rev }); + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("tar", true, { "x", "-C", tmpDir }, tar); + + gitInfo.storePath = store->addToStore(name, tmpDir); + + gitInfo.revCount = std::stoull(runProgram("git", true, { "-C", cacheDir, "rev-list", "--count", gitInfo.rev })); + + nlohmann::json json; + json["storePath"] = gitInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["rev"] = gitInfo.rev; + json["revCount"] = gitInfo.revCount; + + writeFile(storeLink, json.dump()); + + return gitInfo; +} + +static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::string url; + std::optional ref; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "ref") + ref = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to 'fetchGit', at %s", attr.name, *attr.pos); + } + + if (url.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); + + } else + url = state.coerceToString(pos, *args[0], context, false, false); + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto gitInfo = exportGit(state.store, url, ref, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), gitInfo.storePath, PathSet({gitInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.create("rev")), gitInfo.rev); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), gitInfo.shortRev); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), gitInfo.revCount); + v.attrs->sort(); + + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(gitInfo.storePath)); +} + +static RegisterPrimOp r("fetchGit", 1, prim_fetchGit); + +} diff --git a/third_party/nix/src/libexpr/primops/fetchMercurial.cc b/third_party/nix/src/libexpr/primops/fetchMercurial.cc new file mode 100644 index 000000000..a907d0e1c --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fetchMercurial.cc @@ -0,0 +1,219 @@ +#include "primops.hh" +#include "eval-inline.hh" +#include "download.hh" +#include "store-api.hh" +#include "pathlocks.hh" + +#include + +#include + +#include + +using namespace std::string_literals; + +namespace nix { + +struct HgInfo +{ + Path storePath; + std::string branch; + std::string rev; + uint64_t revCount = 0; +}; + +std::regex commitHashRegex("^[0-9a-fA-F]{40}$"); + +HgInfo exportMercurial(ref store, const std::string & uri, + std::string rev, const std::string & name) +{ + if (evalSettings.pureEval && rev == "") + throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); + + if (rev == "" && hasPrefix(uri, "/") && pathExists(uri + "/.hg")) { + + bool clean = runProgram("hg", true, { "status", "-R", uri, "--modified", "--added", "--removed" }) == ""; + + if (!clean) { + + /* This is an unclean working tree. So copy all tracked + files. */ + + printTalkative("copying unclean Mercurial working tree '%s'", uri); + + HgInfo hgInfo; + hgInfo.rev = "0000000000000000000000000000000000000000"; + hgInfo.branch = chomp(runProgram("hg", true, { "branch", "-R", uri })); + + auto files = tokenizeString>( + runProgram("hg", true, { "status", "-R", uri, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s); + + PathFilter filter = [&](const Path & p) -> bool { + assert(hasPrefix(p, uri)); + std::string file(p, uri.size() + 1); + + auto st = lstat(p); + + if (S_ISDIR(st.st_mode)) { + auto prefix = file + "/"; + auto i = files.lower_bound(prefix); + return i != files.end() && hasPrefix(*i, prefix); + } + + return files.count(file); + }; + + hgInfo.storePath = store->addToStore("source", uri, true, htSHA256, filter); + + return hgInfo; + } + } + + if (rev == "") rev = "default"; + + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, uri).to_string(Base32, false)); + + Path stampFile = fmt("%s/.hg/%s.stamp", cacheDir, hashString(htSHA512, rev).to_string(Base32, false)); + + /* If we haven't pulled this repo less than ‘tarball-ttl’ seconds, + do so now. */ + time_t now = time(0); + struct stat st; + if (stat(stampFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now) + { + /* Except that if this is a commit hash that we already have, + we don't have to pull again. */ + if (!(std::regex_match(rev, commitHashRegex) + && pathExists(cacheDir) + && runProgram( + RunOptions("hg", { "log", "-R", cacheDir, "-r", rev, "--template", "1" }) + .killStderr(true)).second == "1")) + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", uri)); + + if (pathExists(cacheDir)) { + try { + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } + catch (ExecError & e) { + string transJournal = cacheDir + "/.hg/store/journal"; + /* hg throws "abandoned transaction" error only if this file exists */ + if (pathExists(transJournal)) { + runProgram("hg", true, { "recover", "-R", cacheDir }); + runProgram("hg", true, { "pull", "-R", cacheDir, "--", uri }); + } else { + throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status))); + } + } + } else { + createDirs(dirOf(cacheDir)); + runProgram("hg", true, { "clone", "--noupdate", "--", uri, cacheDir }); + } + } + + writeFile(stampFile, ""); + } + + auto tokens = tokenizeString>( + runProgram("hg", true, { "log", "-R", cacheDir, "-r", rev, "--template", "{node} {rev} {branch}" })); + assert(tokens.size() == 3); + + HgInfo hgInfo; + hgInfo.rev = tokens[0]; + hgInfo.revCount = std::stoull(tokens[1]); + hgInfo.branch = tokens[2]; + + std::string storeLinkName = hashString(htSHA512, name + std::string("\0"s) + hgInfo.rev).to_string(Base32, false); + Path storeLink = fmt("%s/.hg/%s.link", cacheDir, storeLinkName); + + try { + auto json = nlohmann::json::parse(readFile(storeLink)); + + assert(json["name"] == name && json["rev"] == hgInfo.rev); + + hgInfo.storePath = json["storePath"]; + + if (store->isValidPath(hgInfo.storePath)) { + printTalkative("using cached Mercurial store path '%s'", hgInfo.storePath); + return hgInfo; + } + + } catch (SysError & e) { + if (e.errNo != ENOENT) throw; + } + + Path tmpDir = createTempDir(); + AutoDelete delTmpDir(tmpDir, true); + + runProgram("hg", true, { "archive", "-R", cacheDir, "-r", rev, tmpDir }); + + deletePath(tmpDir + "/.hg_archival.txt"); + + hgInfo.storePath = store->addToStore(name, tmpDir); + + nlohmann::json json; + json["storePath"] = hgInfo.storePath; + json["uri"] = uri; + json["name"] = name; + json["branch"] = hgInfo.branch; + json["rev"] = hgInfo.rev; + json["revCount"] = hgInfo.revCount; + + writeFile(storeLink, json.dump()); + + return hgInfo; +} + +static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + std::string url; + std::string rev; + std::string name = "source"; + PathSet context; + + state.forceValue(*args[0]); + + if (args[0]->type == tAttrs) { + + state.forceAttrs(*args[0], pos); + + for (auto & attr : *args[0]->attrs) { + string n(attr.name); + if (n == "url") + url = state.coerceToString(*attr.pos, *attr.value, context, false, false); + else if (n == "rev") + rev = state.forceStringNoCtx(*attr.value, *attr.pos); + else if (n == "name") + name = state.forceStringNoCtx(*attr.value, *attr.pos); + else + throw EvalError("unsupported argument '%s' to 'fetchMercurial', at %s", attr.name, *attr.pos); + } + + if (url.empty()) + throw EvalError(format("'url' argument required, at %1%") % pos); + + } else + url = state.coerceToString(pos, *args[0], context, false, false); + + // FIXME: git externals probably can be used to bypass the URI + // whitelist. Ah well. + state.checkURI(url); + + auto hgInfo = exportMercurial(state.store, url, rev, name); + + state.mkAttrs(v, 8); + mkString(*state.allocAttr(v, state.sOutPath), hgInfo.storePath, PathSet({hgInfo.storePath})); + mkString(*state.allocAttr(v, state.symbols.create("branch")), hgInfo.branch); + mkString(*state.allocAttr(v, state.symbols.create("rev")), hgInfo.rev); + mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(hgInfo.rev, 0, 12)); + mkInt(*state.allocAttr(v, state.symbols.create("revCount")), hgInfo.revCount); + v.attrs->sort(); + + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(hgInfo.storePath)); +} + +static RegisterPrimOp r("fetchMercurial", 1, prim_fetchMercurial); + +} diff --git a/third_party/nix/src/libexpr/primops/fromTOML.cc b/third_party/nix/src/libexpr/primops/fromTOML.cc new file mode 100644 index 000000000..a84e569e9 --- /dev/null +++ b/third_party/nix/src/libexpr/primops/fromTOML.cc @@ -0,0 +1,90 @@ +#include "primops.hh" +#include "eval-inline.hh" + +#include "cpptoml/cpptoml.h" + +namespace nix { + +static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + using namespace cpptoml; + + auto toml = state.forceStringNoCtx(*args[0], pos); + + std::istringstream tomlStream(toml); + + std::function)> visit; + + visit = [&](Value & v, std::shared_ptr t) { + + if (auto t2 = t->as_table()) { + + size_t size = 0; + for (auto & i : *t2) { (void) i; size++; } + + state.mkAttrs(v, size); + + for (auto & i : *t2) { + auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + + if (auto i2 = i.second->as_table_array()) { + size_t size2 = i2->get().size(); + state.mkList(v2, size2); + for (size_t j = 0; j < size2; ++j) + visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + } + else + visit(v2, i.second); + } + + v.attrs->sort(); + } + + else if (auto t2 = t->as_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); + } + + // Handle cases like 'a = [[{ a = true }]]', which IMHO should be + // parsed as a array containing an array containing a table, + // but instead are parsed as an array containing a table array + // containing a table. + else if (auto t2 = t->as_table_array()) { + size_t size = t2->get().size(); + + state.mkList(v, size); + + for (size_t j = 0; j < size; ++j) + visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); + } + + else if (t->is_value()) { + if (auto val = t->as()) + mkInt(v, val->get()); + else if (auto val = t->as()) + mkFloat(v, val->get()); + else if (auto val = t->as()) + mkBool(v, val->get()); + else if (auto val = t->as()) + mkString(v, val->get()); + else + throw EvalError("unsupported value type in TOML"); + } + + else abort(); + }; + + try { + visit(v, parser(tomlStream).parse()); + } catch (std::runtime_error & e) { + throw EvalError("while parsing a TOML string at %s: %s", pos, e.what()); + } +} + +static RegisterPrimOp r("fromTOML", 1, prim_fromTOML); + +} diff --git a/third_party/nix/src/libexpr/symbol-table.hh b/third_party/nix/src/libexpr/symbol-table.hh new file mode 100644 index 000000000..91faea122 --- /dev/null +++ b/third_party/nix/src/libexpr/symbol-table.hh @@ -0,0 +1,87 @@ +#pragma once + +#include +#include + +#include "types.hh" + +namespace nix { + +/* Symbol table used by the parser and evaluator to represent and look + up identifiers and attributes efficiently. SymbolTable::create() + converts a string into a symbol. Symbols have the property that + they can be compared efficiently (using a pointer equality test), + because the symbol table stores only one copy of each string. */ + +class Symbol +{ +private: + const string * s; // pointer into SymbolTable + Symbol(const string * s) : s(s) { }; + friend class SymbolTable; + +public: + Symbol() : s(0) { }; + + bool operator == (const Symbol & s2) const + { + return s == s2.s; + } + + bool operator != (const Symbol & s2) const + { + return s != s2.s; + } + + bool operator < (const Symbol & s2) const + { + return s < s2.s; + } + + operator const string & () const + { + return *s; + } + + bool set() const + { + return s; + } + + bool empty() const + { + return s->empty(); + } + + friend std::ostream & operator << (std::ostream & str, const Symbol & sym); +}; + +class SymbolTable +{ +private: + typedef std::unordered_set Symbols; + Symbols symbols; + +public: + Symbol create(const string & s) + { + std::pair res = symbols.insert(s); + return Symbol(&*res.first); + } + + size_t size() const + { + return symbols.size(); + } + + size_t totalSize() const; + + template + void dump(T callback) + { + for (auto & s : symbols) + callback(s); + } +}; + +} diff --git a/third_party/nix/src/libexpr/value-to-json.cc b/third_party/nix/src/libexpr/value-to-json.cc new file mode 100644 index 000000000..5fe8570ad --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-json.cc @@ -0,0 +1,100 @@ +#include "value-to-json.hh" +#include "json.hh" +#include "eval-inline.hh" +#include "util.hh" + +#include +#include + + +namespace nix { + +void printValueAsJSON(EvalState & state, bool strict, + Value & v, JSONPlaceholder & out, PathSet & context) +{ + checkInterrupt(); + + if (strict) state.forceValue(v); + + switch (v.type) { + + case tInt: + out.write(v.integer); + break; + + case tBool: + out.write(v.boolean); + break; + + case tString: + copyContext(v, context); + out.write(v.string.s); + break; + + case tPath: + out.write(state.copyPathToStore(context, v.path)); + break; + + case tNull: + out.write(nullptr); + break; + + case tAttrs: { + auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); + if (maybeString) { + out.write(*maybeString); + break; + } + auto i = v.attrs->find(state.sOutPath); + if (i == v.attrs->end()) { + auto obj(out.object()); + StringSet names; + for (auto & j : *v.attrs) + names.insert(j.name); + for (auto & j : names) { + Attr & a(*v.attrs->find(state.symbols.create(j))); + auto placeholder(obj.placeholder(j)); + printValueAsJSON(state, strict, *a.value, placeholder, context); + } + } else + printValueAsJSON(state, strict, *i->value, out, context); + break; + } + + case tList1: case tList2: case tListN: { + auto list(out.list()); + for (unsigned int n = 0; n < v.listSize(); ++n) { + auto placeholder(list.placeholder()); + printValueAsJSON(state, strict, *v.listElems()[n], placeholder, context); + } + break; + } + + case tExternal: + v.external->printValueAsJSON(state, strict, out, context); + break; + + case tFloat: + out.write(v.fpoint); + break; + + default: + throw TypeError(format("cannot convert %1% to JSON") % showType(v)); + } +} + +void printValueAsJSON(EvalState & state, bool strict, + Value & v, std::ostream & str, PathSet & context) +{ + JSONPlaceholder out(str); + printValueAsJSON(state, strict, v, out, context); +} + +void ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, + JSONPlaceholder & out, PathSet & context) const +{ + throw TypeError(format("cannot convert %1% to JSON") % showType()); +} + + +} diff --git a/third_party/nix/src/libexpr/value-to-json.hh b/third_party/nix/src/libexpr/value-to-json.hh new file mode 100644 index 000000000..67fed6487 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-json.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "nixexpr.hh" +#include "eval.hh" + +#include +#include + +namespace nix { + +class JSONPlaceholder; + +void printValueAsJSON(EvalState & state, bool strict, + Value & v, JSONPlaceholder & out, PathSet & context); + +void printValueAsJSON(EvalState & state, bool strict, + Value & v, std::ostream & str, PathSet & context); + +} diff --git a/third_party/nix/src/libexpr/value-to-xml.cc b/third_party/nix/src/libexpr/value-to-xml.cc new file mode 100644 index 000000000..00b1918a8 --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-xml.cc @@ -0,0 +1,178 @@ +#include "value-to-xml.hh" +#include "xml-writer.hh" +#include "eval-inline.hh" +#include "util.hh" + +#include + + +namespace nix { + + +static XMLAttrs singletonAttrs(const string & name, const string & value) +{ + XMLAttrs attrs; + attrs[name] = value; + return attrs; +} + + +static void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen); + + +static void posToXML(XMLAttrs & xmlAttrs, const Pos & pos) +{ + xmlAttrs["path"] = pos.file; + xmlAttrs["line"] = (format("%1%") % pos.line).str(); + xmlAttrs["column"] = (format("%1%") % pos.column).str(); +} + + +static void showAttrs(EvalState & state, bool strict, bool location, + Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) +{ + StringSet names; + + for (auto & i : attrs) + names.insert(i.name); + + for (auto & i : names) { + Attr & a(*attrs.find(state.symbols.create(i))); + + XMLAttrs xmlAttrs; + xmlAttrs["name"] = i; + if (location && a.pos != &noPos) posToXML(xmlAttrs, *a.pos); + + XMLOpenElement _(doc, "attr", xmlAttrs); + printValueAsXML(state, strict, location, + *a.value, doc, context, drvsSeen); + } +} + + +static void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) +{ + checkInterrupt(); + + if (strict) state.forceValue(v); + + switch (v.type) { + + case tInt: + doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); + break; + + case tBool: + doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); + break; + + case tString: + /* !!! show the context? */ + copyContext(v, context); + doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); + break; + + case tPath: + doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + break; + + case tNull: + doc.writeEmptyElement("null"); + break; + + case tAttrs: + if (state.isDerivation(v)) { + XMLAttrs xmlAttrs; + + Bindings::iterator a = v.attrs->find(state.symbols.create("derivation")); + + Path drvPath; + a = v.attrs->find(state.sDrvPath); + if (a != v.attrs->end()) { + if (strict) state.forceValue(*a->value); + if (a->value->type == tString) + xmlAttrs["drvPath"] = drvPath = a->value->string.s; + } + + a = v.attrs->find(state.sOutPath); + if (a != v.attrs->end()) { + if (strict) state.forceValue(*a->value); + if (a->value->type == tString) + xmlAttrs["outPath"] = a->value->string.s; + } + + XMLOpenElement _(doc, "derivation", xmlAttrs); + + if (drvPath != "" && drvsSeen.find(drvPath) == drvsSeen.end()) { + drvsSeen.insert(drvPath); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } else + doc.writeEmptyElement("repeated"); + } + + else { + XMLOpenElement _(doc, "attrs"); + showAttrs(state, strict, location, *v.attrs, doc, context, drvsSeen); + } + + break; + + case tList1: case tList2: case tListN: { + XMLOpenElement _(doc, "list"); + for (unsigned int n = 0; n < v.listSize(); ++n) + printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); + break; + } + + case tLambda: { + XMLAttrs xmlAttrs; + if (location) posToXML(xmlAttrs, v.lambda.fun->pos); + XMLOpenElement _(doc, "function", xmlAttrs); + + if (v.lambda.fun->matchAttrs) { + XMLAttrs attrs; + if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg; + if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1"; + XMLOpenElement _(doc, "attrspat", attrs); + for (auto & i : v.lambda.fun->formals->formals) + doc.writeEmptyElement("attr", singletonAttrs("name", i.name)); + } else + doc.writeEmptyElement("varpat", singletonAttrs("name", v.lambda.fun->arg)); + + break; + } + + case tExternal: + v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); + break; + + case tFloat: + doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); + break; + + default: + doc.writeEmptyElement("unevaluated"); + } +} + + +void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, + bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const +{ + doc.writeEmptyElement("unevaluated"); +} + + +void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, std::ostream & out, PathSet & context) +{ + XMLWriter doc(true, out); + XMLOpenElement root(doc, "expr"); + PathSet drvsSeen; + printValueAsXML(state, strict, location, v, doc, context, drvsSeen); +} + + +} diff --git a/third_party/nix/src/libexpr/value-to-xml.hh b/third_party/nix/src/libexpr/value-to-xml.hh new file mode 100644 index 000000000..97657327e --- /dev/null +++ b/third_party/nix/src/libexpr/value-to-xml.hh @@ -0,0 +1,14 @@ +#pragma once + +#include "nixexpr.hh" +#include "eval.hh" + +#include +#include + +namespace nix { + +void printValueAsXML(EvalState & state, bool strict, bool location, + Value & v, std::ostream & out, PathSet & context); + +} diff --git a/third_party/nix/src/libexpr/value.hh b/third_party/nix/src/libexpr/value.hh new file mode 100644 index 000000000..e1ec87d3b --- /dev/null +++ b/third_party/nix/src/libexpr/value.hh @@ -0,0 +1,274 @@ +#pragma once + +#include "symbol-table.hh" + +#if HAVE_BOEHMGC +#include +#endif + +namespace nix { + + +typedef enum { + tInt = 1, + tBool, + tString, + tPath, + tNull, + tAttrs, + tList1, + tList2, + tListN, + tThunk, + tApp, + tLambda, + tBlackhole, + tPrimOp, + tPrimOpApp, + tExternal, + tFloat +} ValueType; + + +class Bindings; +struct Env; +struct Expr; +struct ExprLambda; +struct PrimOp; +struct PrimOp; +class Symbol; +struct Pos; +class EvalState; +class XMLWriter; +class JSONPlaceholder; + + +typedef int64_t NixInt; +typedef double NixFloat; + +/* External values must descend from ExternalValueBase, so that + * type-agnostic nix functions (e.g. showType) can be implemented + */ +class ExternalValueBase +{ + friend std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); + protected: + /* Print out the value */ + virtual std::ostream & print(std::ostream & str) const = 0; + + public: + /* Return a simple string describing the type */ + virtual string showType() const = 0; + + /* Return a string to be used in builtins.typeOf */ + virtual string typeOf() const = 0; + + /* How much space does this value take up */ + virtual size_t valueSize(std::set & seen) const = 0; + + /* Coerce the value to a string. Defaults to uncoercable, i.e. throws an + * error + */ + virtual string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + + /* Compare to another value of the same type. Defaults to uncomparable, + * i.e. always false. + */ + virtual bool operator==(const ExternalValueBase & b) const; + + /* Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ + virtual void printValueAsJSON(EvalState & state, bool strict, + JSONPlaceholder & out, PathSet & context) const; + + /* Print the value as XML. Defaults to unevaluated */ + virtual void printValueAsXML(EvalState & state, bool strict, bool location, + XMLWriter & doc, PathSet & context, PathSet & drvsSeen) const; + + virtual ~ExternalValueBase() + { + }; +}; + +std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); + + +struct Value +{ + ValueType type; + union + { + NixInt integer; + bool boolean; + + /* Strings in the evaluator carry a so-called `context' which + is a list of strings representing store paths. This is to + allow users to write things like + + "--with-freetype2-library=" + freetype + "/lib" + + where `freetype' is a derivation (or a source to be copied + to the store). If we just concatenated the strings without + keeping track of the referenced store paths, then if the + string is used as a derivation attribute, the derivation + will not have the correct dependencies in its inputDrvs and + inputSrcs. + + The semantics of the context is as follows: when a string + with context C is used as a derivation attribute, then the + derivations in C will be added to the inputDrvs of the + derivation, and the other store paths in C will be added to + the inputSrcs of the derivations. + + For canonicity, the store paths should be in sorted order. */ + struct { + const char * s; + const char * * context; // must be in sorted order + } string; + + const char * path; + Bindings * attrs; + struct { + size_t size; + Value * * elems; + } bigList; + Value * smallList[2]; + struct { + Env * env; + Expr * expr; + } thunk; + struct { + Value * left, * right; + } app; + struct { + Env * env; + ExprLambda * fun; + } lambda; + PrimOp * primOp; + struct { + Value * left, * right; + } primOpApp; + ExternalValueBase * external; + NixFloat fpoint; + }; + + bool isList() const + { + return type == tList1 || type == tList2 || type == tListN; + } + + Value * * listElems() + { + return type == tList1 || type == tList2 ? smallList : bigList.elems; + } + + const Value * const * listElems() const + { + return type == tList1 || type == tList2 ? smallList : bigList.elems; + } + + size_t listSize() const + { + return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + } +}; + + +/* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ +static inline void clearValue(Value & v) +{ + v.app.left = v.app.right = 0; +} + + +static inline void mkInt(Value & v, NixInt n) +{ + clearValue(v); + v.type = tInt; + v.integer = n; +} + + +static inline void mkFloat(Value & v, NixFloat n) +{ + clearValue(v); + v.type = tFloat; + v.fpoint = n; +} + + +static inline void mkBool(Value & v, bool b) +{ + clearValue(v); + v.type = tBool; + v.boolean = b; +} + + +static inline void mkNull(Value & v) +{ + clearValue(v); + v.type = tNull; +} + + +static inline void mkApp(Value & v, Value & left, Value & right) +{ + v.type = tApp; + v.app.left = &left; + v.app.right = &right; +} + + +static inline void mkPrimOpApp(Value & v, Value & left, Value & right) +{ + v.type = tPrimOpApp; + v.app.left = &left; + v.app.right = &right; +} + + +static inline void mkStringNoCopy(Value & v, const char * s) +{ + v.type = tString; + v.string.s = s; + v.string.context = 0; +} + + +static inline void mkString(Value & v, const Symbol & s) +{ + mkStringNoCopy(v, ((const string &) s).c_str()); +} + + +void mkString(Value & v, const char * s); + + +static inline void mkPathNoCopy(Value & v, const char * s) +{ + clearValue(v); + v.type = tPath; + v.path = s; +} + + +void mkPath(Value & v, const char * s); + + +/* Compute the size in bytes of the given value, including all values + and environments reachable from it. Static expressions (Exprs) are + not included. */ +size_t valueSize(Value & v); + + +#if HAVE_BOEHMGC +typedef std::vector > ValueVector; +typedef std::map, gc_allocator > > ValueMap; +#else +typedef std::vector ValueVector; +typedef std::map ValueMap; +#endif + + +} diff --git a/third_party/nix/src/libmain/common-args.cc b/third_party/nix/src/libmain/common-args.cc new file mode 100644 index 000000000..9e1d7cee6 --- /dev/null +++ b/third_party/nix/src/libmain/common-args.cc @@ -0,0 +1,56 @@ +#include "common-args.hh" +#include "globals.hh" + +namespace nix { + +MixCommonArgs::MixCommonArgs(const string & programName) + : programName(programName) +{ + mkFlag() + .longName("verbose") + .shortName('v') + .description("increase verbosity level") + .handler([]() { verbosity = (Verbosity) (verbosity + 1); }); + + mkFlag() + .longName("quiet") + .description("decrease verbosity level") + .handler([]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }); + + mkFlag() + .longName("debug") + .description("enable debug output") + .handler([]() { verbosity = lvlDebug; }); + + mkFlag() + .longName("option") + .labels({"name", "value"}) + .description("set a Nix configuration option (overriding nix.conf)") + .arity(2) + .handler([](std::vector ss) { + try { + globalConfig.set(ss[0], ss[1]); + } catch (UsageError & e) { + warn(e.what()); + } + }); + + mkFlag() + .longName("max-jobs") + .shortName('j') + .label("jobs") + .description("maximum number of parallel builds") + .handler([=](std::string s) { + settings.set("max-jobs", s); + }); + + std::string cat = "config"; + globalConfig.convertToArgs(*this, cat); + + // Backward compatibility hack: nix-env already had a --system flag. + if (programName == "nix-env") longFlags.erase("system"); + + hiddenCategories.insert(cat); +} + +} diff --git a/third_party/nix/src/libmain/common-args.hh b/third_party/nix/src/libmain/common-args.hh new file mode 100644 index 000000000..a4de3dccf --- /dev/null +++ b/third_party/nix/src/libmain/common-args.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "args.hh" + +namespace nix { + +struct MixCommonArgs : virtual Args +{ + string programName; + MixCommonArgs(const string & programName); +}; + +struct MixDryRun : virtual Args +{ + bool dryRun = false; + + MixDryRun() + { + mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + } +}; + +struct MixJSON : virtual Args +{ + bool json = false; + + MixJSON() + { + mkFlag(0, "json", "produce JSON output", &json); + } +}; + +} diff --git a/third_party/nix/src/libmain/local.mk b/third_party/nix/src/libmain/local.mk new file mode 100644 index 000000000..0c80f5a0a --- /dev/null +++ b/third_party/nix/src/libmain/local.mk @@ -0,0 +1,15 @@ +libraries += libmain + +libmain_NAME = libnixmain + +libmain_DIR := $(d) + +libmain_SOURCES := $(wildcard $(d)/*.cc) + +libmain_LDFLAGS = $(OPENSSL_LIBS) + +libmain_LIBS = libstore libutil + +libmain_ALLOW_UNDEFINED = 1 + +$(eval $(call install-file-in, $(d)/nix-main.pc, $(prefix)/lib/pkgconfig, 0644)) diff --git a/third_party/nix/src/libmain/nix-main.pc.in b/third_party/nix/src/libmain/nix-main.pc.in new file mode 100644 index 000000000..37b03dcd4 --- /dev/null +++ b/third_party/nix/src/libmain/nix-main.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixmain +Cflags: -I${includedir}/nix -std=c++17 diff --git a/third_party/nix/src/libmain/shared.cc b/third_party/nix/src/libmain/shared.cc new file mode 100644 index 000000000..d3dbfbc44 --- /dev/null +++ b/third_party/nix/src/libmain/shared.cc @@ -0,0 +1,381 @@ +#include "globals.hh" +#include "shared.hh" +#include "store-api.hh" +#include "util.hh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + + +namespace nix { + + +static bool gcWarning = true; + +void printGCWarning() +{ + if (!gcWarning) return; + static bool haveWarned = false; + warnOnce(haveWarned, + "you did not specify '--add-root'; " + "the result might be removed by the garbage collector"); +} + + +void printMissing(ref store, const PathSet & paths, Verbosity lvl) +{ + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl); +} + + +void printMissing(ref store, const PathSet & willBuild, + const PathSet & willSubstitute, const PathSet & unknown, + unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl) +{ + if (!willBuild.empty()) { + printMsg(lvl, "these derivations will be built:"); + Paths sorted = store->topoSortPaths(willBuild); + reverse(sorted.begin(), sorted.end()); + for (auto & i : sorted) + printMsg(lvl, fmt(" %s", i)); + } + + if (!willSubstitute.empty()) { + printMsg(lvl, fmt("these paths will be fetched (%.2f MiB download, %.2f MiB unpacked):", + downloadSize / (1024.0 * 1024.0), + narSize / (1024.0 * 1024.0))); + for (auto & i : willSubstitute) + printMsg(lvl, fmt(" %s", i)); + } + + if (!unknown.empty()) { + printMsg(lvl, fmt("don't know how to build these paths%s:", + (settings.readOnlyMode ? " (may be caused by read-only store access)" : ""))); + for (auto & i : unknown) + printMsg(lvl, fmt(" %s", i)); + } +} + + +string getArg(const string & opt, + Strings::iterator & i, const Strings::iterator & end) +{ + ++i; + if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + return *i; +} + + +#if OPENSSL_VERSION_NUMBER < 0x10101000L +/* OpenSSL is not thread-safe by default - it will randomly crash + unless the user supplies a mutex locking function. So let's do + that. */ +static std::vector opensslLocks; + +static void opensslLockCallback(int mode, int type, const char * file, int line) +{ + if (mode & CRYPTO_LOCK) + opensslLocks[type].lock(); + else + opensslLocks[type].unlock(); +} +#endif + + +static void sigHandler(int signo) { } + + +void initNix() +{ + /* Turn on buffering for cerr. */ +#if HAVE_PUBSETBUF + static char buf[1024]; + std::cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); +#endif + +#if OPENSSL_VERSION_NUMBER < 0x10101000L + /* Initialise OpenSSL locking. */ + opensslLocks = std::vector(CRYPTO_num_locks()); + CRYPTO_set_locking_callback(opensslLockCallback); +#endif + + loadConfFile(); + + startSignalHandlerThread(); + + /* Reset SIGCHLD to its default. */ + struct sigaction act; + sigemptyset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, 0)) + throw SysError("resetting SIGCHLD"); + + /* Install a dummy SIGUSR1 handler for use with pthread_kill(). */ + act.sa_handler = sigHandler; + if (sigaction(SIGUSR1, &act, 0)) throw SysError("handling SIGUSR1"); + +#if __APPLE__ + /* HACK: on darwin, we need can’t use sigprocmask with SIGWINCH. + * Instead, add a dummy sigaction handler, and signalHandlerThread + * can handle the rest. */ + struct sigaction sa; + sa.sa_handler = sigHandler; + if (sigaction(SIGWINCH, &sa, 0)) throw SysError("handling SIGWINCH"); +#endif + + /* Register a SIGSEGV handler to detect stack overflows. */ + detectStackOverflow(); + + /* There is no privacy in the Nix system ;-) At least not for + now. In particular, store objects should be readable by + everybody. */ + umask(0022); + + /* Initialise the PRNG. */ + struct timeval tv; + gettimeofday(&tv, 0); + srandom(tv.tv_usec); + + /* On macOS, don't use the per-session TMPDIR (as set e.g. by + sshd). This breaks build users because they don't have access + to the TMPDIR, in particular in ‘nix-store --serve’. */ +#if __APPLE__ + if (getuid() == 0 && hasPrefix(getEnv("TMPDIR"), "/var/folders/")) + unsetenv("TMPDIR"); +#endif +} + + +LegacyArgs::LegacyArgs(const std::string & programName, + std::function parseArg) + : MixCommonArgs(programName), parseArg(parseArg) +{ + mkFlag() + .longName("no-build-output") + .shortName('Q') + .description("do not show build output") + .set(&settings.verboseBuild, false); + + mkFlag() + .longName("keep-failed") + .shortName('K') + .description("keep temporary directories of failed builds") + .set(&(bool&) settings.keepFailed, true); + + mkFlag() + .longName("keep-going") + .shortName('k') + .description("keep going after a build fails") + .set(&(bool&) settings.keepGoing, true); + + mkFlag() + .longName("fallback") + .description("build from source if substitution fails") + .set(&(bool&) settings.tryFallback, true); + + auto intSettingAlias = [&](char shortName, const std::string & longName, + const std::string & description, const std::string & dest) { + mkFlag(shortName, longName, description, [=](unsigned int n) { + settings.set(dest, std::to_string(n)); + }); + }; + + intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); + intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); + intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + + mkFlag(0, "readonly-mode", "do not write to the Nix store", + &settings.readOnlyMode); + + mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + &gcWarning, false); + + mkFlag() + .longName("store") + .label("store-uri") + .description("URI of the Nix store to use") + .dest(&(std::string&) settings.storeUri); +} + + +bool LegacyArgs::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (MixCommonArgs::processFlag(pos, end)) return true; + bool res = parseArg(pos, end); + if (res) ++pos; + return res; +} + + +bool LegacyArgs::processArgs(const Strings & args, bool finish) +{ + if (args.empty()) return true; + assert(args.size() == 1); + Strings ss(args); + auto pos = ss.begin(); + if (!parseArg(pos, ss.end())) + throw UsageError(format("unexpected argument '%1%'") % args.front()); + return true; +} + + +void parseCmdLine(int argc, char * * argv, + std::function parseArg) +{ + parseCmdLine(baseNameOf(argv[0]), argvToStrings(argc, argv), parseArg); +} + + +void parseCmdLine(const string & programName, const Strings & args, + std::function parseArg) +{ + LegacyArgs(programName, parseArg).parseCmdline(args); +} + + +void printVersion(const string & programName) +{ + std::cout << format("%1% (Nix) %2%") % programName % nixVersion << std::endl; + if (verbosity > lvlInfo) { + Strings cfg; +#if HAVE_BOEHMGC + cfg.push_back("gc"); +#endif +#if HAVE_SODIUM + cfg.push_back("signed-caches"); +#endif + std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; + std::cout << "Configuration file: " << settings.nixConfDir + "/nix.conf" << "\n"; + std::cout << "Store directory: " << settings.nixStore << "\n"; + std::cout << "State directory: " << settings.nixStateDir << "\n"; + } + throw Exit(); +} + + +void showManPage(const string & name) +{ + restoreSignals(); + setenv("MANPATH", settings.nixManDir.c_str(), 1); + execlp("man", "man", name.c_str(), nullptr); + throw SysError(format("command 'man %1%' failed") % name.c_str()); +} + + +int handleExceptions(const string & programName, std::function fun) +{ + ReceiveInterrupts receiveInterrupts; // FIXME: need better place for this + + string error = ANSI_RED "error:" ANSI_NORMAL " "; + try { + try { + fun(); + } catch (...) { + /* Subtle: we have to make sure that any `interrupted' + condition is discharged before we reach printMsg() + below, since otherwise it will throw an (uncaught) + exception. */ + setInterruptThrown(); + throw; + } + } catch (Exit & e) { + return e.status; + } catch (UsageError & e) { + printError( + format(error + "%1%\nTry '%2% --help' for more information.") + % e.what() % programName); + return 1; + } catch (BaseError & e) { + printError(format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + if (e.prefix() != "" && !settings.showTrace) + printError("(use '--show-trace' to show detailed location information)"); + return e.status; + } catch (std::bad_alloc & e) { + printError(error + "out of memory"); + return 1; + } catch (std::exception & e) { + printError(error + e.what()); + return 1; + } + + return 0; +} + + +RunPager::RunPager() +{ + if (!isatty(STDOUT_FILENO)) return; + char * pager = getenv("NIX_PAGER"); + if (!pager) pager = getenv("PAGER"); + if (pager && ((string) pager == "" || (string) pager == "cat")) return; + + Pipe toPager; + toPager.create(); + + pid = startProcess([&]() { + if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + if (!getenv("LESS")) + setenv("LESS", "FRSXMK", 1); + restoreSignals(); + if (pager) + execl("/bin/sh", "sh", "-c", pager, nullptr); + execlp("pager", "pager", nullptr); + execlp("less", "less", nullptr); + execlp("more", "more", nullptr); + throw SysError(format("executing '%1%'") % pager); + }); + + pid.setKillSignal(SIGINT); + + if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); +} + + +RunPager::~RunPager() +{ + try { + if (pid != -1) { + std::cout.flush(); + close(STDOUT_FILENO); + pid.wait(); + } + } catch (...) { + ignoreException(); + } +} + + +string showBytes(unsigned long long bytes) +{ + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +} + + +PrintFreed::~PrintFreed() +{ + if (show) + std::cout << format("%1% store paths deleted, %2% freed\n") + % results.paths.size() + % showBytes(results.bytesFreed); +} + +Exit::~Exit() { } + +} diff --git a/third_party/nix/src/libmain/shared.hh b/third_party/nix/src/libmain/shared.hh new file mode 100644 index 000000000..8e4861232 --- /dev/null +++ b/third_party/nix/src/libmain/shared.hh @@ -0,0 +1,126 @@ +#pragma once + +#include "util.hh" +#include "args.hh" +#include "common-args.hh" + +#include + +#include + + +namespace nix { + +class Exit : public std::exception +{ +public: + int status; + Exit() : status(0) { } + Exit(int status) : status(status) { } + virtual ~Exit(); +}; + +int handleExceptions(const string & programName, std::function fun); + +/* Don't forget to call initPlugins() after settings are initialized! */ +void initNix(); + +void parseCmdLine(int argc, char * * argv, + std::function parseArg); + +void parseCmdLine(const string & programName, const Strings & args, + std::function parseArg); + +void printVersion(const string & programName); + +/* Ugh. No better place to put this. */ +void printGCWarning(); + +class Store; + +void printMissing(ref store, const PathSet & paths, Verbosity lvl = lvlInfo); + +void printMissing(ref store, const PathSet & willBuild, + const PathSet & willSubstitute, const PathSet & unknown, + unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo); + +string getArg(const string & opt, + Strings::iterator & i, const Strings::iterator & end); + +template N getIntArg(const string & opt, + Strings::iterator & i, const Strings::iterator & end, bool allowUnit) +{ + ++i; + if (i == end) throw UsageError(format("'%1%' requires an argument") % opt); + string s = *i; + N multiplier = 1; + if (allowUnit && !s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError(format("invalid unit specifier '%1%'") % u); + s.resize(s.size() - 1); + } + } + N n; + if (!string2Int(s, n)) + throw UsageError(format("'%1%' requires an integer argument") % opt); + return n * multiplier; +} + + +struct LegacyArgs : public MixCommonArgs +{ + std::function parseArg; + + LegacyArgs(const std::string & programName, + std::function parseArg); + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + + +/* Show the manual page for the specified program. */ +void showManPage(const string & name); + +/* The constructor of this class starts a pager if stdout is a + terminal and $PAGER is set. Stdout is redirected to the pager. */ +class RunPager +{ +public: + RunPager(); + ~RunPager(); + +private: + Pid pid; +}; + +extern volatile ::sig_atomic_t blockInt; + + +/* GC helpers. */ + +string showBytes(unsigned long long bytes); + +struct GCResults; + +struct PrintFreed +{ + bool show; + const GCResults & results; + PrintFreed(bool show, const GCResults & results) + : show(show), results(results) { } + ~PrintFreed(); +}; + + +/* Install a SIGSEGV handler to detect stack overflows. */ +void detectStackOverflow(); + + +} diff --git a/third_party/nix/src/libmain/stack.cc b/third_party/nix/src/libmain/stack.cc new file mode 100644 index 000000000..e6224de7d --- /dev/null +++ b/third_party/nix/src/libmain/stack.cc @@ -0,0 +1,71 @@ +#include "types.hh" + +#include +#include +#include + +#include +#include + +namespace nix { + + +static void sigsegvHandler(int signo, siginfo_t * info, void * ctx) +{ + /* Detect stack overflows by comparing the faulting address with + the stack pointer. Unfortunately, getting the stack pointer is + not portable. */ + bool haveSP = true; + char * sp = 0; +#if defined(__x86_64__) && defined(REG_RSP) + sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_RSP]; +#elif defined(REG_ESP) + sp = (char *) ((ucontext_t *) ctx)->uc_mcontext.gregs[REG_ESP]; +#else + haveSP = false; +#endif + + if (haveSP) { + ptrdiff_t diff = (char *) info->si_addr - sp; + if (diff < 0) diff = -diff; + if (diff < 4096) { + char msg[] = "error: stack overflow (possible infinite recursion)\n"; + [[gnu::unused]] auto res = write(2, msg, strlen(msg)); + _exit(1); // maybe abort instead? + } + } + + /* Restore default behaviour (i.e. segfault and dump core). */ + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + if (sigaction(SIGSEGV, &act, 0)) abort(); +} + + +void detectStackOverflow() +{ +#if defined(SA_SIGINFO) && defined (SA_ONSTACK) + /* Install a SIGSEGV handler to detect stack overflows. This + requires an alternative stack, otherwise the signal cannot be + delivered when we're out of stack space. */ + stack_t stack; + stack.ss_size = 4096 * 4 + MINSIGSTKSZ; + static auto stackBuf = std::make_unique>(stack.ss_size); + stack.ss_sp = stackBuf->data(); + if (!stack.ss_sp) throw Error("cannot allocate alternative stack"); + stack.ss_flags = 0; + if (sigaltstack(&stack, 0) == -1) throw SysError("cannot set alternative stack"); + + struct sigaction act; + sigfillset(&act.sa_mask); + act.sa_sigaction = sigsegvHandler; + act.sa_flags = SA_SIGINFO | SA_ONSTACK; + if (sigaction(SIGSEGV, &act, 0)) + throw SysError("resetting SIGSEGV"); +#endif +} + + +} diff --git a/third_party/nix/src/libstore/binary-cache-store.cc b/third_party/nix/src/libstore/binary-cache-store.cc new file mode 100644 index 000000000..10cde8704 --- /dev/null +++ b/third_party/nix/src/libstore/binary-cache-store.cc @@ -0,0 +1,361 @@ +#include "archive.hh" +#include "binary-cache-store.hh" +#include "compression.hh" +#include "derivations.hh" +#include "fs-accessor.hh" +#include "globals.hh" +#include "nar-info.hh" +#include "sync.hh" +#include "remote-fs-accessor.hh" +#include "nar-info-disk-cache.hh" +#include "nar-accessor.hh" +#include "json.hh" + +#include + +#include + +namespace nix { + +BinaryCacheStore::BinaryCacheStore(const Params & params) + : Store(params) +{ + if (secretKeyFile != "") + secretKey = std::unique_ptr(new SecretKey(readFile(secretKeyFile))); + + StringSink sink; + sink << narVersionMagic1; + narMagic = *sink.s; +} + +void BinaryCacheStore::init() +{ + std::string cacheInfoFile = "nix-cache-info"; + + auto cacheInfo = getFile(cacheInfoFile); + if (!cacheInfo) { + upsertFile(cacheInfoFile, "StoreDir: " + storeDir + "\n", "text/x-nix-cache-info"); + } else { + for (auto & line : tokenizeString(*cacheInfo, "\n")) { + size_t colon = line.find(':'); + if (colon == std::string::npos) continue; + auto name = line.substr(0, colon); + auto value = trim(line.substr(colon + 1, std::string::npos)); + if (name == "StoreDir") { + if (value != storeDir) + throw Error(format("binary cache '%s' is for Nix stores with prefix '%s', not '%s'") + % getUri() % value % storeDir); + } else if (name == "WantMassQuery") { + wantMassQuery_ = value == "1"; + } else if (name == "Priority") { + string2Int(value, priority); + } + } + } +} + +void BinaryCacheStore::getFile(const std::string & path, + Callback> callback) noexcept +{ + try { + callback(getFile(path)); + } catch (...) { callback.rethrow(); } +} + +void BinaryCacheStore::getFile(const std::string & path, Sink & sink) +{ + std::promise> promise; + getFile(path, + {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + auto data = promise.get_future().get(); + sink((unsigned char *) data->data(), data->size()); +} + +std::shared_ptr BinaryCacheStore::getFile(const std::string & path) +{ + StringSink sink; + try { + getFile(path, sink); + } catch (NoSuchBinaryCacheFile &) { + return nullptr; + } + return sink.s; +} + +Path BinaryCacheStore::narInfoFileFor(const Path & storePath) +{ + assertStorePath(storePath); + return storePathToHash(storePath) + ".narinfo"; +} + +void BinaryCacheStore::writeNarInfo(ref narInfo) +{ + auto narInfoFile = narInfoFileFor(narInfo->path); + + upsertFile(narInfoFile, narInfo->to_string(), "text/x-nix-narinfo"); + + auto hashPart = storePathToHash(narInfo->path); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, std::shared_ptr(narInfo)); + } + + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr(narInfo)); +} + +void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref & nar, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) +{ + if (!repair && isValidPath(info.path)) return; + + /* Verify that all references are valid. This may do some .narinfo + reads, but typically they'll already be cached. */ + for (auto & ref : info.references) + try { + if (ref != info.path) + queryPathInfo(ref); + } catch (InvalidPath &) { + throw Error(format("cannot add '%s' to the binary cache because the reference '%s' is not valid") + % info.path % ref); + } + + assert(nar->compare(0, narMagic.size(), narMagic) == 0); + + auto narInfo = make_ref(info); + + narInfo->narSize = nar->size(); + narInfo->narHash = hashString(htSHA256, *nar); + + if (info.narHash && info.narHash != narInfo->narHash) + throw Error(format("refusing to copy corrupted path '%1%' to binary cache") % info.path); + + auto accessor_ = std::dynamic_pointer_cast(accessor); + + /* Optionally write a JSON file containing a listing of the + contents of the NAR. */ + if (writeNARListing) { + std::ostringstream jsonOut; + + { + JSONObject jsonRoot(jsonOut); + jsonRoot.attr("version", 1); + + auto narAccessor = makeNarAccessor(nar); + + if (accessor_) + accessor_->addToCache(info.path, *nar, narAccessor); + + { + auto res = jsonRoot.placeholder("root"); + listNar(res, narAccessor, "", true); + } + } + + upsertFile(storePathToHash(info.path) + ".ls", jsonOut.str(), "application/json"); + } + + else { + if (accessor_) + accessor_->addToCache(info.path, *nar, makeNarAccessor(nar)); + } + + /* Compress the NAR. */ + narInfo->compression = compression; + auto now1 = std::chrono::steady_clock::now(); + auto narCompressed = compress(compression, *nar, parallelCompression); + auto now2 = std::chrono::steady_clock::now(); + narInfo->fileHash = hashString(htSHA256, *narCompressed); + narInfo->fileSize = narCompressed->size(); + + auto duration = std::chrono::duration_cast(now2 - now1).count(); + printMsg(lvlTalkative, format("copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache") + % narInfo->path % narInfo->narSize + % ((1.0 - (double) narCompressed->size() / nar->size()) * 100.0) + % duration); + + /* Atomically write the NAR file. */ + narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar" + + (compression == "xz" ? ".xz" : + compression == "bzip2" ? ".bz2" : + compression == "br" ? ".br" : + ""); + if (repair || !fileExists(narInfo->url)) { + stats.narWrite++; + upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar"); + } else + stats.narWriteAverted++; + + stats.narWriteBytes += nar->size(); + stats.narWriteCompressedBytes += narCompressed->size(); + stats.narWriteCompressionTimeMs += duration; + + /* Atomically write the NAR info file.*/ + if (secretKey) narInfo->sign(*secretKey); + + writeNarInfo(narInfo); + + stats.narInfoWrite++; +} + +bool BinaryCacheStore::isValidPathUncached(const Path & storePath) +{ + // FIXME: this only checks whether a .narinfo with a matching hash + // part exists. So ‘f4kb...-foo’ matches ‘f4kb...-bar’, even + // though they shouldn't. Not easily fixed. + return fileExists(narInfoFileFor(storePath)); +} + +void BinaryCacheStore::narFromPath(const Path & storePath, Sink & sink) +{ + auto info = queryPathInfo(storePath).cast(); + + uint64_t narSize = 0; + + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + narSize += len; + }); + + auto decompressor = makeDecompressionSink(info->compression, wrapperSink); + + try { + getFile(info->url, *decompressor); + } catch (NoSuchBinaryCacheFile & e) { + throw SubstituteGone(e.what()); + } + + decompressor->finish(); + + stats.narRead++; + //stats.narReadCompressedBytes += nar->size(); // FIXME + stats.narReadBytes += narSize; +} + +void BinaryCacheStore::queryPathInfoUncached(const Path & storePath, + Callback> callback) noexcept +{ + auto uri = getUri(); + auto act = std::make_shared(*logger, lvlTalkative, actQueryPathInfo, + fmt("querying info about '%s' on '%s'", storePath, uri), Logger::Fields{storePath, uri}); + PushActivity pact(act->id); + + auto narInfoFile = narInfoFileFor(storePath); + + auto callbackPtr = std::make_shared(std::move(callback)); + + getFile(narInfoFile, + {[=](std::future> fut) { + try { + auto data = fut.get(); + + if (!data) return (*callbackPtr)(nullptr); + + stats.narInfoRead++; + + (*callbackPtr)((std::shared_ptr) + std::make_shared(*this, *data, narInfoFile)); + + (void) act; // force Activity into this lambda to ensure it stays alive + } catch (...) { + callbackPtr->rethrow(); + } + }}); +} + +Path BinaryCacheStore::addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + // FIXME: some cut&paste from LocalStore::addToStore(). + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + Hash h; + if (recursive) { + dumpPath(srcPath, sink, filter); + h = hashString(hashAlgo, *sink.s); + } else { + auto s = readFile(srcPath); + dumpString(s, sink); + h = hashString(hashAlgo, s); + } + + ValidPathInfo info; + info.path = makeFixedOutputPath(recursive, h, name); + + addToStore(info, sink.s, repair, CheckSigs, nullptr); + + return info.path; +} + +Path BinaryCacheStore::addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) +{ + ValidPathInfo info; + info.path = computeStorePathForText(name, s, references); + info.references = references; + + if (repair || !isValidPath(info.path)) { + StringSink sink; + dumpString(s, sink); + addToStore(info, sink.s, repair, CheckSigs, nullptr); + } + + return info.path; +} + +ref BinaryCacheStore::getFSAccessor() +{ + return make_ref(ref(shared_from_this()), localNarCache); +} + +void BinaryCacheStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + /* Note: this is inherently racy since there is no locking on + binary caches. In particular, with S3 this unreliable, even + when addSignatures() is called sequentially on a path, because + S3 might return an outdated cached version. */ + + auto narInfo = make_ref((NarInfo &) *queryPathInfo(storePath)); + + narInfo->sigs.insert(sigs.begin(), sigs.end()); + + auto narInfoFile = narInfoFileFor(narInfo->path); + + writeNarInfo(narInfo); +} + +std::shared_ptr BinaryCacheStore::getBuildLog(const Path & path) +{ + Path drvPath; + + if (isDerivation(path)) + drvPath = path; + else { + try { + auto info = queryPathInfo(path); + // FIXME: add a "Log" field to .narinfo + if (info->deriver == "") return nullptr; + drvPath = info->deriver; + } catch (InvalidPath &) { + return nullptr; + } + } + + auto logPath = "log/" + baseNameOf(drvPath); + + debug("fetching build log from binary cache '%s/%s'", getUri(), logPath); + + return getFile(logPath); +} + +} diff --git a/third_party/nix/src/libstore/binary-cache-store.hh b/third_party/nix/src/libstore/binary-cache-store.hh new file mode 100644 index 000000000..af108880c --- /dev/null +++ b/third_party/nix/src/libstore/binary-cache-store.hh @@ -0,0 +1,115 @@ +#pragma once + +#include "crypto.hh" +#include "store-api.hh" + +#include "pool.hh" + +#include + +namespace nix { + +struct NarInfo; + +class BinaryCacheStore : public Store +{ +public: + + const Setting compression{this, "xz", "compression", "NAR compression method ('xz', 'bzip2', or 'none')"}; + const Setting writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"}; + const Setting secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"}; + const Setting localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"}; + const Setting parallelCompression{this, false, "parallel-compression", + "enable multi-threading compression, available for xz only currently"}; + +private: + + std::unique_ptr secretKey; + +protected: + + BinaryCacheStore(const Params & params); + +public: + + virtual bool fileExists(const std::string & path) = 0; + + virtual void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) = 0; + + /* Note: subclasses must implement at least one of the two + following getFile() methods. */ + + /* Dump the contents of the specified file to a sink. */ + virtual void getFile(const std::string & path, Sink & sink); + + /* Fetch the specified file and call the specified callback with + the result. A subclass may implement this asynchronously. */ + virtual void getFile(const std::string & path, + Callback> callback) noexcept; + + std::shared_ptr getFile(const std::string & path); + +protected: + + bool wantMassQuery_ = false; + int priority = 50; + +public: + + virtual void init(); + +private: + + std::string narMagic; + + std::string narInfoFileFor(const Path & storePath); + + void writeNarInfo(ref narInfo); + +public: + + bool isValidPathUncached(const Path & path) override; + + void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override; + + Path queryPathFromHashPart(const string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + bool wantMassQuery() override { return wantMassQuery_; } + + void addToStore(const ValidPathInfo & info, const ref & nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) override; + + Path addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override; + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) override; + + void narFromPath(const Path & path, Sink & sink) override; + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { unsupported("buildDerivation"); } + + void ensurePath(const Path & path) override + { unsupported("ensurePath"); } + + ref getFSAccessor() override; + + void addSignatures(const Path & storePath, const StringSet & sigs) override; + + std::shared_ptr getBuildLog(const Path & path) override; + + int getPriority() override { return priority; } + +}; + +MakeError(NoSuchBinaryCacheFile, Error); + +} diff --git a/third_party/nix/src/libstore/build.cc b/third_party/nix/src/libstore/build.cc new file mode 100644 index 000000000..539e1ea71 --- /dev/null +++ b/third_party/nix/src/libstore/build.cc @@ -0,0 +1,4708 @@ +#include "references.hh" +#include "pathlocks.hh" +#include "globals.hh" +#include "local-store.hh" +#include "util.hh" +#include "archive.hh" +#include "affinity.hh" +#include "builtins.hh" +#include "download.hh" +#include "finally.hh" +#include "compression.hh" +#include "json.hh" +#include "nar-info.hh" +#include "parsed-derivations.hh" +#include "machines.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +/* Includes required for chroot support. */ +#if __linux__ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HAVE_SECCOMP +#include +#endif +#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +#endif + +#if HAVE_STATVFS +#include +#endif + +#include + + +namespace nix { + +using std::map; + + +static string pathNullDevice = "/dev/null"; + + +/* Forward definition. */ +class Worker; +struct HookInstance; + + +/* A pointer to a goal. */ +class Goal; +class DerivationGoal; +typedef std::shared_ptr GoalPtr; +typedef std::weak_ptr WeakGoalPtr; + +struct CompareGoalPtrs { + bool operator() (const GoalPtr & a, const GoalPtr & b) const; +}; + +/* Set of goals. */ +typedef set Goals; +typedef list WeakGoals; + +/* A map of paths to goals (and the other way around). */ +typedef map WeakGoalMap; + + + +class Goal : public std::enable_shared_from_this +{ +public: + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; + +protected: + + /* Backlink to the worker. */ + Worker & worker; + + /* Goals that this goal is waiting for. */ + Goals waitees; + + /* Goals waiting for this one to finish. Must use weak pointers + here to prevent cycles. */ + WeakGoals waiters; + + /* Number of goals we are/were waiting for that have failed. */ + unsigned int nrFailed; + + /* Number of substitution goals we are/were waiting for that + failed because there are no substituters. */ + unsigned int nrNoSubstituters; + + /* Number of substitution goals we are/were waiting for that + failed because othey had unsubstitutable references. */ + unsigned int nrIncompleteClosure; + + /* Name of this goal for debugging purposes. */ + string name; + + /* Whether the goal is finished. */ + ExitCode exitCode; + + Goal(Worker & worker) : worker(worker) + { + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + exitCode = ecBusy; + } + + virtual ~Goal() + { + trace("goal destroyed"); + } + +public: + virtual void work() = 0; + + void addWaitee(GoalPtr waitee); + + virtual void waiteeDone(GoalPtr waitee, ExitCode result); + + virtual void handleChildOutput(int fd, const string & data) + { + abort(); + } + + virtual void handleEOF(int fd) + { + abort(); + } + + void trace(const FormatOrString & fs); + + string getName() + { + return name; + } + + ExitCode getExitCode() + { + return exitCode; + } + + /* Callback in case of a timeout. It should wake up its waiters, + get rid of any running child processes that are being monitored + by the worker (important!), etc. */ + virtual void timedOut() = 0; + + virtual string key() = 0; + +protected: + + virtual void amDone(ExitCode result); +}; + + +bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { + string s1 = a->key(); + string s2 = b->key(); + return s1 < s2; +} + + +typedef std::chrono::time_point steady_time_point; + + +/* A mapping used to remember for each child process to what goal it + belongs, and file descriptors for receiving log data and output + path creation commands. */ +struct Child +{ + WeakGoalPtr goal; + Goal * goal2; // ugly hackery + set fds; + bool respectTimeouts; + bool inBuildSlot; + steady_time_point lastOutput; /* time we last got output on stdout/stderr */ + steady_time_point timeStarted; +}; + + +/* The worker class. */ +class Worker +{ +private: + + /* Note: the worker should only have strong pointers to the + top-level goals. */ + + /* The top-level goals of the worker. */ + Goals topGoals; + + /* Goals that are ready to do some work. */ + WeakGoals awake; + + /* Goals waiting for a build slot. */ + WeakGoals wantingToBuild; + + /* Child processes currently running. */ + std::list children; + + /* Number of build slots occupied. This includes local builds and + substitutions but not remote builds via the build hook. */ + unsigned int nrLocalBuilds; + + /* Maps used to prevent multiple instantiations of a goal for the + same derivation / path. */ + WeakGoalMap derivationGoals; + WeakGoalMap substitutionGoals; + + /* Goals waiting for busy paths to be unlocked. */ + WeakGoals waitingForAnyGoal; + + /* Goals sleeping for a few seconds (polling a lock). */ + WeakGoals waitingForAWhile; + + /* Last time the goals in `waitingForAWhile' where woken up. */ + steady_time_point lastWokenUp; + + /* Cache for pathContentsGood(). */ + std::map pathContentsGoodCache; + +public: + + const Activity act; + const Activity actDerivations; + const Activity actSubstitutions; + + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; + + /* Set if at least one derivation had a timeout. */ + bool timedOut; + + /* Set if at least one derivation fails with a hash mismatch. */ + bool hashMismatch; + + /* Set if at least one derivation is not deterministic in check mode. */ + bool checkMismatch; + + LocalStore & store; + + std::unique_ptr hook; + + uint64_t expectedBuilds = 0; + uint64_t doneBuilds = 0; + uint64_t failedBuilds = 0; + uint64_t runningBuilds = 0; + + uint64_t expectedSubstitutions = 0; + uint64_t doneSubstitutions = 0; + uint64_t failedSubstitutions = 0; + uint64_t runningSubstitutions = 0; + uint64_t expectedDownloadSize = 0; + uint64_t doneDownloadSize = 0; + uint64_t expectedNarSize = 0; + uint64_t doneNarSize = 0; + + /* Whether to ask the build hook if it can build a derivation. If + it answers with "decline-permanently", we don't try again. */ + bool tryBuildHook = true; + + Worker(LocalStore & store); + ~Worker(); + + /* Make a goal (with caching). */ + GoalPtr makeDerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, BuildMode buildMode = bmNormal); + std::shared_ptr makeBasicDerivationGoal(const Path & drvPath, + const BasicDerivation & drv, BuildMode buildMode = bmNormal); + GoalPtr makeSubstitutionGoal(const Path & storePath, RepairFlag repair = NoRepair); + + /* Remove a dead goal. */ + void removeGoal(GoalPtr goal); + + /* Wake up a goal (i.e., there is something for it to do). */ + void wakeUp(GoalPtr goal); + + /* Return the number of local build and substitution processes + currently running (but not remote builds via the build + hook). */ + unsigned int getNrLocalBuilds(); + + /* Registers a running child process. `inBuildSlot' means that + the process counts towards the jobs limit. */ + void childStarted(GoalPtr goal, const set & fds, + bool inBuildSlot, bool respectTimeouts); + + /* Unregisters a running child process. `wakeSleepers' should be + false if there is no sense in waking up goals that are sleeping + because they can't run yet (e.g., there is no free build slot, + or the hook would still say `postpone'). */ + void childTerminated(Goal * goal, bool wakeSleepers = true); + + /* Put `goal' to sleep until a build slot becomes available (which + might be right away). */ + void waitForBuildSlot(GoalPtr goal); + + /* Wait for any goal to finish. Pretty indiscriminate way to + wait for some resource that some other goal is holding. */ + void waitForAnyGoal(GoalPtr goal); + + /* Wait for a few seconds and then retry this goal. Used when + waiting for a lock held by another process. This kind of + polling is inefficient, but POSIX doesn't really provide a way + to wait for multiple locks in the main select() loop. */ + void waitForAWhile(GoalPtr goal); + + /* Loop until the specified top-level goals have finished. */ + void run(const Goals & topGoals); + + /* Wait for input to become available. */ + void waitForInput(); + + unsigned int exitStatus(); + + /* Check whether the given valid path exists and has the right + contents. */ + bool pathContentsGood(const Path & path); + + void markContentsGood(const Path & path); + + void updateProgress() + { + actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds); + actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions); + act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize); + act.setExpected(actCopyPath, expectedNarSize + doneNarSize); + } +}; + + +////////////////////////////////////////////////////////////////////// + + +void addToWeakGoals(WeakGoals & goals, GoalPtr p) +{ + // FIXME: necessary? + // FIXME: O(n) + for (auto & i : goals) + if (i.lock() == p) return; + goals.push_back(p); +} + + +void Goal::addWaitee(GoalPtr waitee) +{ + waitees.insert(waitee); + addToWeakGoals(waitee->waiters, shared_from_this()); +} + + +void Goal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + assert(waitees.find(waitee) != waitees.end()); + waitees.erase(waitee); + + trace(format("waitee '%1%' done; %2% left") % + waitee->name % waitees.size()); + + if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; + + if (result == ecNoSubstituters) ++nrNoSubstituters; + + if (result == ecIncompleteClosure) ++nrIncompleteClosure; + + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (auto & goal : waitees) { + WeakGoals waiters2; + for (auto & j : goal->waiters) + if (j.lock() != shared_from_this()) waiters2.push_back(j); + goal->waiters = waiters2; + } + waitees.clear(); + + worker.wakeUp(shared_from_this()); + } +} + + +void Goal::amDone(ExitCode result) +{ + trace("done"); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); + exitCode = result; + for (auto & i : waiters) { + GoalPtr goal = i.lock(); + if (goal) goal->waiteeDone(shared_from_this(), result); + } + waiters.clear(); + worker.removeGoal(shared_from_this()); +} + + +void Goal::trace(const FormatOrString & fs) +{ + debug("%1%: %2%", name, fs.s); +} + + + +////////////////////////////////////////////////////////////////////// + + +/* Common initialisation performed in child processes. */ +static void commonChildInit(Pipe & logPipe) +{ + restoreSignals(); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Dup the write side of the logger pipe into stderr. */ + if (dup2(logPipe.writeSide.get(), STDERR_FILENO) == -1) + throw SysError("cannot pipe standard error into log file"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError(format("cannot open '%1%'") % pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +void handleDiffHook(uid_t uid, uid_t gid, Path tryA, Path tryB, Path drvPath, Path tmpDir) +{ + auto diffHook = settings.diffHook; + if (diffHook != "" && settings.runDiffHook) { + try { + RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir}); + diffHookOptions.searchPath = true; + diffHookOptions.uid = uid; + diffHookOptions.gid = gid; + diffHookOptions.chdir = "/"; + + auto diffRes = runProgram(diffHookOptions); + if (!statusOk(diffRes.first)) + throw ExecError(diffRes.first, fmt("diff-hook program '%1%' %2%", diffHook, statusToString(diffRes.first))); + + if (diffRes.second != "") + printError(chomp(diffRes.second)); + } catch (Error & error) { + printError("diff hook execution failed: %s", error.what()); + } + } +} + +////////////////////////////////////////////////////////////////////// + + +class UserLock +{ +private: + /* POSIX locks suck. If we have a lock on a file, and we open and + close that file again (without closing the original file + descriptor), we lose the lock. So we have to be *very* careful + not to open a lock file on which we are holding a lock. */ + static Sync lockedPaths_; + + Path fnUserLock; + AutoCloseFD fdUserLock; + + string user; + uid_t uid; + gid_t gid; + std::vector supplementaryGIDs; + +public: + UserLock(); + ~UserLock(); + + void kill(); + + string getUser() { return user; } + uid_t getUID() { assert(uid); return uid; } + uid_t getGID() { assert(gid); return gid; } + std::vector getSupplementaryGIDs() { return supplementaryGIDs; } + + bool enabled() { return uid != 0; } + +}; + + +Sync UserLock::lockedPaths_; + + +UserLock::UserLock() +{ + assert(settings.buildUsersGroup != ""); + + /* Get the members of the build-users-group. */ + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + throw Error(format("the group '%1%' specified in 'build-users-group' does not exist") + % settings.buildUsersGroup); + gid = gr->gr_gid; + + /* Copy the result of getgrnam. */ + Strings users; + for (char * * p = gr->gr_mem; *p; ++p) { + debug(format("found build user '%1%'") % *p); + users.push_back(*p); + } + + if (users.empty()) + throw Error(format("the build users group '%1%' has no members") + % settings.buildUsersGroup); + + /* Find a user account that isn't currently in use for another + build. */ + for (auto & i : users) { + debug(format("trying user '%1%'") % i); + + struct passwd * pw = getpwnam(i.c_str()); + if (!pw) + throw Error(format("the user '%1%' in the group '%2%' does not exist") + % i % settings.buildUsersGroup); + + createDirs(settings.nixStateDir + "/userpool"); + + fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str(); + + { + auto lockedPaths(lockedPaths_.lock()); + if (lockedPaths->count(fnUserLock)) + /* We already have a lock on this one. */ + continue; + lockedPaths->insert(fnUserLock); + } + + try { + + AutoCloseFD fd = open(fnUserLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fd) + throw SysError(format("opening user lock '%1%'") % fnUserLock); + + if (lockFile(fd.get(), ltWrite, false)) { + fdUserLock = std::move(fd); + user = i; + uid = pw->pw_uid; + + /* Sanity check... */ + if (uid == getuid() || uid == geteuid()) + throw Error(format("the Nix user should not be a member of '%1%'") + % settings.buildUsersGroup); + +#if __linux__ + /* Get the list of supplementary groups of this build user. This + is usually either empty or contains a group such as "kvm". */ + supplementaryGIDs.resize(10); + int ngroups = supplementaryGIDs.size(); + int err = getgrouplist(pw->pw_name, pw->pw_gid, + supplementaryGIDs.data(), &ngroups); + if (err == -1) + throw Error(format("failed to get list of supplementary groups for '%1%'") % pw->pw_name); + + supplementaryGIDs.resize(ngroups); +#endif + + return; + } + + } catch (...) { + lockedPaths_.lock()->erase(fnUserLock); + } + } + + throw Error(format("all build users are currently in use; " + "consider creating additional users and adding them to the '%1%' group") + % settings.buildUsersGroup); +} + + +UserLock::~UserLock() +{ + auto lockedPaths(lockedPaths_.lock()); + assert(lockedPaths->count(fnUserLock)); + lockedPaths->erase(fnUserLock); +} + + +void UserLock::kill() +{ + killUser(uid); +} + + +////////////////////////////////////////////////////////////////////// + + +struct HookInstance +{ + /* Pipes for talking to the build hook. */ + Pipe toHook; + + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* The process ID of the hook. */ + Pid pid; + + FdSink sink; + + std::map activities; + + HookInstance(); + + ~HookInstance(); +}; + + +HookInstance::HookInstance() +{ + debug("starting build hook '%s'", settings.buildHook); + + /* Create a pipe to get the output of the child. */ + fromHook.create(); + + /* Create the communication pipes. */ + toHook.create(); + + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + + /* Fork the hook. */ + pid = startProcess([&]() { + + commonChildInit(fromHook); + + if (chdir("/") == -1) throw SysError("changing into /"); + + /* Dup the communication pipes. */ + if (dup2(toHook.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); + + /* Use fd 4 for the builder's stdout/stderr. */ + if (dup2(builderOut.writeSide.get(), 4) == -1) + throw SysError("dupping builder's stdout/stderr"); + + /* Hack: pass the read side of that fd to allow build-remote + to read SSH error messages. */ + if (dup2(builderOut.readSide.get(), 5) == -1) + throw SysError("dupping builder's stdout/stderr"); + + Strings args = { + baseNameOf(settings.buildHook), + std::to_string(verbosity), + }; + + execv(settings.buildHook.get().c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing '%s'", settings.buildHook); + }); + + pid.setSeparatePG(true); + fromHook.writeSide = -1; + toHook.readSide = -1; + + sink = FdSink(toHook.writeSide.get()); + std::map settings; + globalConfig.getSettings(settings); + for (auto & setting : settings) + sink << 1 << setting.first << setting.second.value; + sink << 0; +} + + +HookInstance::~HookInstance() +{ + try { + toHook.writeSide = -1; + if (pid != -1) pid.kill(); + } catch (...) { + ignoreException(); + } +} + + +////////////////////////////////////////////////////////////////////// + + +typedef map StringRewrites; + + +std::string rewriteStrings(std::string s, const StringRewrites & rewrites) +{ + for (auto & i : rewrites) { + size_t j = 0; + while ((j = s.find(i.first, j)) != string::npos) + s.replace(j, i.first.size(), i.second); + } + return s; +} + + +////////////////////////////////////////////////////////////////////// + + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + +class SubstitutionGoal; + +class DerivationGoal : public Goal +{ +private: + /* Whether to use an on-disk .drv file. */ + bool useDerivation; + + /* The path of the derivation. */ + Path drvPath; + + /* The specific outputs that we need to build. Empty means all of + them. */ + StringSet wantedOutputs; + + /* Whether additional wanted outputs have been added. */ + bool needRestart = false; + + /* Whether to retry substituting the outputs after building the + inputs. */ + bool retrySubstitution; + + /* The derivation stored at drvPath. */ + std::unique_ptr drv; + + std::unique_ptr parsedDrv; + + /* The remainder is state held during the build. */ + + /* Locks on the output paths. */ + PathLocks outputLocks; + + /* All input paths (that is, the union of FS closures of the + immediate input paths). */ + PathSet inputPaths; + + /* Referenceable paths (i.e., input and output paths). */ + PathSet allPaths; + + /* Outputs that are already valid. If we're repairing, these are + the outputs that are valid *and* not corrupt. */ + PathSet validPaths; + + /* Outputs that are corrupt or not valid. */ + PathSet missingPaths; + + /* User selected for running the builder. */ + std::unique_ptr buildUser; + + /* The process ID of the builder. */ + Pid pid; + + /* The temporary directory. */ + Path tmpDir; + + /* The path of the temporary directory in the sandbox. */ + Path tmpDirInSandbox; + + /* File descriptor for the log file. */ + AutoCloseFD fdLogFile; + std::shared_ptr logFileSink, logSink; + + /* Number of bytes received from the builder's stdout/stderr. */ + unsigned long logSize; + + /* The most recent log lines. */ + std::list logTail; + + std::string currentLogLine; + size_t currentLogLinePos = 0; // to handle carriage return + + std::string currentHookLine; + + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + + /* Pipe for synchronising updates to the builder user namespace. */ + Pipe userNamespaceSync; + + /* The build hook. */ + std::unique_ptr hook; + + /* Whether we're currently doing a chroot build. */ + bool useChroot = false; + + Path chrootRootDir; + + /* RAII object to delete the chroot directory. */ + std::shared_ptr autoDelChroot; + + /* Whether this is a fixed-output derivation. */ + bool fixedOutput; + + /* Whether to run the build in a private network namespace. */ + bool privateNetwork = false; + + typedef void (DerivationGoal::*GoalState)(); + GoalState state; + + /* Stuff we need to pass to initChild(). */ + struct ChrootPath { + Path source; + bool optional; + ChrootPath(Path source = "", bool optional = false) + : source(source), optional(optional) + { } + }; + typedef map DirsInChroot; // maps target path to source path + DirsInChroot dirsInChroot; + + typedef map Environment; + Environment env; + +#if __APPLE__ + typedef string SandboxProfile; + SandboxProfile additionalSandboxProfile; +#endif + + /* Hash rewriting. */ + StringRewrites inputRewrites, outputRewrites; + typedef map RedirectedOutputs; + RedirectedOutputs redirectedOutputs; + + BuildMode buildMode; + + /* If we're repairing without a chroot, there may be outputs that + are valid but corrupt. So we redirect these outputs to + temporary paths. */ + PathSet redirectedBadOutputs; + + BuildResult result; + + /* The current round, if we're building multiple times. */ + size_t curRound = 1; + + size_t nrRounds; + + /* Path registration info from the previous round, if we're + building multiple times. Since this contains the hash, it + allows us to compare whether two rounds produced the same + result. */ + std::map prevInfos; + + const uid_t sandboxUid = 1000; + const gid_t sandboxGid = 100; + + const static Path homeDir; + + std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; + + std::unique_ptr act; + + std::map builderActivities; + + /* The remote machine on which we're building. */ + std::string machineName; + +public: + DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + Worker & worker, BuildMode buildMode = bmNormal); + DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + Worker & worker, BuildMode buildMode = bmNormal); + ~DerivationGoal(); + + /* Whether we need to perform hash rewriting if there are valid output paths. */ + bool needsHashRewrite(); + + void timedOut() override; + + string key() override + { + /* Ensure that derivations get built in order of their name, + i.e. a derivation named "aardvark" always comes before + "baboon". And substitution goals always happen before + derivation goals (due to "b$"). */ + return "b$" + storePathToName(drvPath) + "$" + drvPath; + } + + void work() override; + + Path getDrvPath() + { + return drvPath; + } + + /* Add wanted outputs to an already existing derivation goal. */ + void addWantedOutputs(const StringSet & outputs); + + BuildResult getResult() { return result; } + +private: + /* The states. */ + void getDerivation(); + void loadDerivation(); + void haveDerivation(); + void outputsSubstituted(); + void closureRepaired(); + void inputsRealised(); + void tryToBuild(); + void buildDone(); + + /* Is the build hook willing to perform the build? */ + HookReply tryBuildHook(); + + /* Start building a derivation. */ + void startBuilder(); + + /* Fill in the environment for the builder. */ + void initEnv(); + + /* Setup tmp dir location. */ + void initTmpDir(); + + /* Write a JSON file containing the derivation attributes. */ + void writeStructuredAttrs(); + + /* Make a file owned by the builder. */ + void chownToBuilder(const Path & path); + + /* Run the builder's process. */ + void runChild(); + + friend int childEntry(void *); + + /* Check that the derivation outputs all exist and register them + as valid. */ + void registerOutputs(); + + /* Check that an output meets the requirements specified by the + 'outputChecks' attribute (or the legacy + '{allowed,disallowed}{References,Requisites}' attributes). */ + void checkOutputs(const std::map & outputs); + + /* Open a log file and a pipe to it. */ + Path openLogFile(); + + /* Close the log file. */ + void closeLogFile(); + + /* Delete the temporary directory, if we have one. */ + void deleteTmpDir(bool force); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; + void flushLine(); + + /* Return the set of (in)valid paths. */ + PathSet checkPathValidity(bool returnValid, bool checkHash); + + /* Abort the goal if `path' failed to build. */ + bool pathFailed(const Path & path); + + /* Forcibly kill the child process, if any. */ + void killChild(); + + Path addHashRewrite(const Path & path); + + void repairClosure(); + + void amDone(ExitCode result) override + { + Goal::amDone(result); + } + + void done(BuildResult::Status status, const string & msg = ""); + + PathSet exportReferences(PathSet storePaths); +}; + + +const Path DerivationGoal::homeDir = "/homeless-shelter"; + + +DerivationGoal::DerivationGoal(const Path & drvPath, const StringSet & wantedOutputs, + Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(true) + , drvPath(drvPath) + , wantedOutputs(wantedOutputs) + , buildMode(buildMode) +{ + state = &DerivationGoal::getDerivation; + name = (format("building of '%1%'") % drvPath).str(); + trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); +} + + +DerivationGoal::DerivationGoal(const Path & drvPath, const BasicDerivation & drv, + Worker & worker, BuildMode buildMode) + : Goal(worker) + , useDerivation(false) + , drvPath(drvPath) + , buildMode(buildMode) +{ + this->drv = std::unique_ptr(new BasicDerivation(drv)); + state = &DerivationGoal::haveDerivation; + name = (format("building of %1%") % showPaths(drv.outputPaths())).str(); + trace("created"); + + mcExpectedBuilds = std::make_unique>(worker.expectedBuilds); + worker.updateProgress(); + + /* Prevent the .chroot directory from being + garbage-collected. (See isActiveTempFile() in gc.cc.) */ + worker.store.addTempRoot(drvPath); +} + + +DerivationGoal::~DerivationGoal() +{ + /* Careful: we should never ever throw an exception from a + destructor. */ + try { killChild(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreException(); } + try { closeLogFile(); } catch (...) { ignoreException(); } +} + + +inline bool DerivationGoal::needsHashRewrite() +{ +#if __linux__ + return !useChroot; +#else + /* Darwin requires hash rewriting even when sandboxing is enabled. */ + return true; +#endif +} + + +void DerivationGoal::killChild() +{ + if (pid != -1) { + worker.childTerminated(this); + + if (buildUser) { + /* If we're using a build user, then there is a tricky + race condition: if we kill the build user before the + child has done its setuid() to the build user uid, then + it won't be killed, and we'll potentially lock up in + pid.wait(). So also send a conventional kill to the + child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + buildUser->kill(); + pid.wait(); + } else + pid.kill(); + + assert(pid == -1); + } + + hook.reset(); +} + + +void DerivationGoal::timedOut() +{ + killChild(); + done(BuildResult::TimedOut); +} + + +void DerivationGoal::work() +{ + (this->*state)(); +} + + +void DerivationGoal::addWantedOutputs(const StringSet & outputs) +{ + /* If we already want all outputs, there is nothing to do. */ + if (wantedOutputs.empty()) return; + + if (outputs.empty()) { + wantedOutputs.clear(); + needRestart = true; + } else + for (auto & i : outputs) + if (wantedOutputs.find(i) == wantedOutputs.end()) { + wantedOutputs.insert(i); + needRestart = true; + } +} + + +void DerivationGoal::getDerivation() +{ + trace("init"); + + /* The first thing to do is to make sure that the derivation + exists. If it doesn't, it may be created through a + substitute. */ + if (buildMode == bmNormal && worker.store.isValidPath(drvPath)) { + loadDerivation(); + return; + } + + addWaitee(worker.makeSubstitutionGoal(drvPath)); + + state = &DerivationGoal::loadDerivation; +} + + +void DerivationGoal::loadDerivation() +{ + trace("loading derivation"); + + if (nrFailed != 0) { + printError(format("cannot build missing derivation '%1%'") % drvPath); + done(BuildResult::MiscFailure); + return; + } + + /* `drvPath' should already be a root, but let's be on the safe + side: if the user forgot to make it a root, we wouldn't want + things being garbage collected while we're busy. */ + worker.store.addTempRoot(drvPath); + + assert(worker.store.isValidPath(drvPath)); + + /* Get the derivation. */ + drv = std::unique_ptr(new Derivation(worker.store.derivationFromPath(drvPath))); + + haveDerivation(); +} + + +void DerivationGoal::haveDerivation() +{ + trace("have derivation"); + + retrySubstitution = false; + + for (auto & i : drv->outputs) + worker.store.addTempRoot(i.second.path); + + /* Check what outputs paths are not already valid. */ + PathSet invalidOutputs = checkPathValidity(false, buildMode == bmRepair); + + /* If they are all valid, then we're done. */ + if (invalidOutputs.size() == 0 && buildMode == bmNormal) { + done(BuildResult::AlreadyValid); + return; + } + + parsedDrv = std::make_unique(drvPath, *drv); + + /* We are first going to try to create the invalid output paths + through substitutes. If that doesn't work, we'll build + them. */ + if (settings.useSubstitutes && parsedDrv->substitutesAllowed()) + for (auto & i : invalidOutputs) + addWaitee(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + outputsSubstituted(); + else + state = &DerivationGoal::outputsSubstituted; +} + + +void DerivationGoal::outputsSubstituted() +{ + trace("all outputs substituted (maybe)"); + + if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { + done(BuildResult::TransientFailure, (format("some substitutes for the outputs of derivation '%1%' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ") % drvPath).str()); + return; + } + + /* If the substitutes form an incomplete closure, then we should + build the dependencies of this derivation, but after that, we + can still use the substitutes for this derivation itself. */ + if (nrIncompleteClosure > 0) retrySubstitution = true; + + nrFailed = nrNoSubstituters = nrIncompleteClosure = 0; + + if (needRestart) { + needRestart = false; + haveDerivation(); + return; + } + + auto nrInvalid = checkPathValidity(false, buildMode == bmRepair).size(); + if (buildMode == bmNormal && nrInvalid == 0) { + done(BuildResult::Substituted); + return; + } + if (buildMode == bmRepair && nrInvalid == 0) { + repairClosure(); + return; + } + if (buildMode == bmCheck && nrInvalid > 0) + throw Error(format("some outputs of '%1%' are not valid, so checking is not possible") % drvPath); + + /* Otherwise, at least one of the output paths could not be + produced using a substitute. So we have to build instead. */ + + /* Make sure checkPathValidity() from now on checks all + outputs. */ + wantedOutputs = PathSet(); + + /* The inputs must be built before we can build this goal. */ + if (useDerivation) + for (auto & i : dynamic_cast(drv.get())->inputDrvs) + addWaitee(worker.makeDerivationGoal(i.first, i.second, buildMode == bmRepair ? bmRepair : bmNormal)); + + for (auto & i : drv->inputSrcs) { + if (worker.store.isValidPath(i)) continue; + if (!settings.useSubstitutes) + throw Error(format("dependency '%1%' of '%2%' does not exist, and substitution is disabled") + % i % drvPath); + addWaitee(worker.makeSubstitutionGoal(i)); + } + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; +} + + +void DerivationGoal::repairClosure() +{ + /* If we're repairing, we now know that our own outputs are valid. + Now check whether the other paths in the outputs closure are + good. If not, then start derivation goals for the derivations + that produced those outputs. */ + + /* Get the output closure. */ + PathSet outputClosure; + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + worker.store.computeFSClosure(i.second.path, outputClosure); + } + + /* Filter out our own outputs (which we have already checked). */ + for (auto & i : drv->outputs) + outputClosure.erase(i.second.path); + + /* Get all dependencies of this derivation so that we know which + derivation is responsible for which path in the output + closure. */ + PathSet inputClosure; + if (useDerivation) worker.store.computeFSClosure(drvPath, inputClosure); + std::map outputsToDrv; + for (auto & i : inputClosure) + if (isDerivation(i)) { + Derivation drv = worker.store.derivationFromPath(i); + for (auto & j : drv.outputs) + outputsToDrv[j.second.path] = i; + } + + /* Check each path (slow!). */ + PathSet broken; + for (auto & i : outputClosure) { + if (worker.pathContentsGood(i)) continue; + printError(format("found corrupted or missing path '%1%' in the output closure of '%2%'") % i % drvPath); + Path drvPath2 = outputsToDrv[i]; + if (drvPath2 == "") + addWaitee(worker.makeSubstitutionGoal(i, Repair)); + else + addWaitee(worker.makeDerivationGoal(drvPath2, PathSet(), bmRepair)); + } + + if (waitees.empty()) { + done(BuildResult::AlreadyValid); + return; + } + + state = &DerivationGoal::closureRepaired; +} + + +void DerivationGoal::closureRepaired() +{ + trace("closure repaired"); + if (nrFailed > 0) + throw Error(format("some paths in the output closure of derivation '%1%' could not be repaired") % drvPath); + done(BuildResult::AlreadyValid); +} + + +void DerivationGoal::inputsRealised() +{ + trace("all inputs realised"); + + if (nrFailed != 0) { + if (!useDerivation) + throw Error(format("some dependencies of '%1%' are missing") % drvPath); + printError( + format("cannot build derivation '%1%': %2% dependencies couldn't be built") + % drvPath % nrFailed); + done(BuildResult::DependencyFailed); + return; + } + + if (retrySubstitution) { + haveDerivation(); + return; + } + + /* Gather information necessary for computing the closure and/or + running the build hook. */ + + /* The outputs are referenceable paths. */ + for (auto & i : drv->outputs) { + debug(format("building path '%1%'") % i.second.path); + allPaths.insert(i.second.path); + } + + /* Determine the full set of input paths. */ + + /* First, the input derivations. */ + if (useDerivation) + for (auto & i : dynamic_cast(drv.get())->inputDrvs) { + /* Add the relevant output closures of the input derivation + `i' as input paths. Only add the closures of output paths + that are specified as inputs. */ + assert(worker.store.isValidPath(i.first)); + Derivation inDrv = worker.store.derivationFromPath(i.first); + for (auto & j : i.second) + if (inDrv.outputs.find(j) != inDrv.outputs.end()) + worker.store.computeFSClosure(inDrv.outputs[j].path, inputPaths); + else + throw Error( + format("derivation '%1%' requires non-existent output '%2%' from input derivation '%3%'") + % drvPath % j % i.first); + } + + /* Second, the input sources. */ + worker.store.computeFSClosure(drv->inputSrcs, inputPaths); + + debug(format("added input paths %1%") % showPaths(inputPaths)); + + allPaths.insert(inputPaths.begin(), inputPaths.end()); + + /* Is this a fixed-output derivation? */ + fixedOutput = drv->isFixedOutput(); + + /* Don't repeat fixed-output derivations since they're already + verified by their output hash.*/ + nrRounds = fixedOutput ? 1 : settings.buildRepeat + 1; + + /* Okay, try to build. Note that here we don't wait for a build + slot to become available, since we don't need one if there is a + build hook. */ + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + + result = BuildResult(); +} + + +void DerivationGoal::tryToBuild() +{ + trace("trying to build"); + + /* Obtain locks on all output paths. The locks are automatically + released when we exit this function or Nix crashes. If we + can't acquire the lock, then continue; hopefully some other + goal can start a build, and if not, the main loop will sleep a + few seconds and then retry this goal. */ + PathSet lockFiles; + for (auto & outPath : drv->outputPaths()) + lockFiles.insert(worker.store.toRealPath(outPath)); + + if (!outputLocks.lockPaths(lockFiles, "", false)) { + worker.waitForAWhile(shared_from_this()); + return; + } + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + validPaths = checkPathValidity(true, buildMode == bmRepair); + if (buildMode != bmCheck && validPaths.size() == drv->outputs.size()) { + debug(format("skipping build of derivation '%1%', someone beat us to it") % drvPath); + outputLocks.setDeletion(true); + done(BuildResult::AlreadyValid); + return; + } + + missingPaths = drv->outputPaths(); + if (buildMode != bmCheck) + for (auto & i : validPaths) missingPaths.erase(i); + + /* If any of the outputs already exist but are not valid, delete + them. */ + for (auto & i : drv->outputs) { + Path path = i.second.path; + if (worker.store.isValidPath(path)) continue; + debug(format("removing invalid path '%1%'") % path); + deletePath(worker.store.toRealPath(path)); + } + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. Also, check and repair modes are only + supported for local builds. */ + bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(); + + auto started = [&]() { + auto msg = fmt( + buildMode == bmRepair ? "repairing outputs of '%s'" : + buildMode == bmCheck ? "checking outputs of '%s'" : + nrRounds > 1 ? "building '%s' (round %d/%d)" : + "building '%s'", drvPath, curRound, nrRounds); + fmt("building '%s'", drvPath); + if (hook) msg += fmt(" on '%s'", machineName); + act = std::make_unique(*logger, lvlInfo, actBuild, msg, + Logger::Fields{drvPath, hook ? machineName : "", curRound, nrRounds}); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); + }; + + /* Is the build hook willing to accept this job? */ + if (!buildLocally) { + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + result.startTime = time(0); // inexact + state = &DerivationGoal::buildDone; + started(); + return; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } + } + + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + worker.waitForBuildSlot(shared_from_this()); + outputLocks.unlock(); + return; + } + + try { + + /* Okay, we have to build. */ + startBuilder(); + + } catch (BuildError & e) { + printError(e.msg()); + outputLocks.unlock(); + buildUser.reset(); + worker.permanentFailure = true; + done(BuildResult::InputRejected, e.msg()); + return; + } + + /* This state will be reached when we get EOF on the child's + log pipe. */ + state = &DerivationGoal::buildDone; + + started(); +} + + +void replaceValidPath(const Path & storePath, const Path tmpPath) +{ + /* We can't atomically replace storePath (the original) with + tmpPath (the replacement), so we have to move it out of the + way first. We'd better not be interrupted here, because if + we're repairing (say) Glibc, we end up with a broken system. */ + Path oldPath = (format("%1%.old-%2%-%3%") % storePath % getpid() % random()).str(); + if (pathExists(storePath)) + rename(storePath.c_str(), oldPath.c_str()); + if (rename(tmpPath.c_str(), storePath.c_str()) == -1) + throw SysError(format("moving '%1%' to '%2%'") % tmpPath % storePath); + deletePath(oldPath); +} + + +MakeError(NotDeterministic, BuildError) + + +void DerivationGoal::buildDone() +{ + trace("build done"); + + /* Release the build user at the end of this function. We don't do + it right away because we don't want another build grabbing this + uid and then messing around with our output. */ + Finally releaseBuildUser([&]() { buildUser.reset(); }); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = hook ? hook->pid.kill() : pid.kill(); + + debug(format("builder process for '%1%' finished") % drvPath); + + result.timesBuilt++; + result.stopTime = time(0); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ + if (hook) { + hook->builderOut.readSide = -1; + hook->fromHook.readSide = -1; + } else + builderOut.readSide = -1; + + /* Close the log file. */ + closeLogFile(); + + /* When running under a build user, make sure that all processes + running under that uid are gone. This is to prevent a + malicious user from leaving behind a process that keeps files + open and modifies them after they have been chown'ed to + root. */ + if (buildUser) buildUser->kill(); + + bool diskFull = false; + + try { + + /* Check the exit status. */ + if (!statusOk(status)) { + + /* Heuristically check whether the build failure may have + been caused by a disk full condition. We have no way + of knowing whether the build actually got an ENOSPC. + So instead, check if the disk is (nearly) full now. If + so, we don't mark this build as a permanent failure. */ +#if HAVE_STATVFS + unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable + struct statvfs st; + if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 && + (unsigned long long) st.f_bavail * st.f_bsize < required) + diskFull = true; + if (statvfs(tmpDir.c_str(), &st) == 0 && + (unsigned long long) st.f_bavail * st.f_bsize < required) + diskFull = true; +#endif + + deleteTmpDir(false); + + /* Move paths out of the chroot for easier debugging of + build failures. */ + if (useChroot && buildMode == bmNormal) + for (auto & i : missingPaths) + if (pathExists(chrootRootDir + i)) + rename((chrootRootDir + i).c_str(), i.c_str()); + + std::string msg = (format("builder for '%1%' %2%") + % drvPath % statusToString(status)).str(); + + if (!settings.verboseBuild && !logTail.empty()) { + msg += (format("; last %d log lines:") % logTail.size()).str(); + for (auto & line : logTail) + msg += "\n " + line; + } + + if (diskFull) + msg += "\nnote: build failure may have been caused by lack of free disk space"; + + throw BuildError(msg); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + registerOutputs(); + + if (settings.postBuildHook != "") { + Activity act(*logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{drvPath}); + PushActivity pact(act.id); + auto outputPaths = drv->outputPaths(); + std::map hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", drvPath); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", outputPaths))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (const unsigned char * data, size_t len) override { + for (size_t i = 0; i < len; i++) { + auto c = data[i]; + + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + if (settings.verboseBuild) { + printError("post-build-hook: " + currentLine); + } else { + act.result(resPostBuildLogLine, currentLine); + } + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); + } + + if (buildMode == bmCheck) { + done(BuildResult::Built); + return; + } + + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(i.second); + + /* Delete the chroot (if we were using one). */ + autoDelChroot.reset(); /* this runs the destructor */ + + deleteTmpDir(true); + + /* Repeat the build if necessary. */ + if (curRound++ < nrRounds) { + outputLocks.unlock(); + state = &DerivationGoal::tryToBuild; + worker.wakeUp(shared_from_this()); + return; + } + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + } catch (BuildError & e) { + printError(e.msg()); + + outputLocks.unlock(); + + BuildResult::Status st = BuildResult::MiscFailure; + + if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) + st = BuildResult::TimedOut; + + else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { + } + + else { + st = + dynamic_cast(&e) ? BuildResult::NotDeterministic : + statusOk(status) ? BuildResult::OutputRejected : + fixedOutput || diskFull ? BuildResult::TransientFailure : + BuildResult::PermanentFailure; + } + + done(st, e.msg()); + return; + } + + done(BuildResult::Built); +} + + +HookReply DerivationGoal::tryBuildHook() +{ + if (!worker.tryBuildHook || !useDerivation) return rpDecline; + + if (!worker.hook) + worker.hook = std::make_unique(); + + try { + + /* Send the request to the hook. */ + worker.hook->sink + << "try" + << (worker.getNrLocalBuilds() < settings.maxBuildJobs ? 1 : 0) + << drv->platform + << drvPath + << parsedDrv->getRequiredSystemFeatures(); + worker.hook->sink.flush(); + + /* Read the first line of input, which should be a word indicating + whether the hook wishes to perform the build. */ + string reply; + while (true) { + string s = readLine(worker.hook->fromHook.readSide.get()); + if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true)) + ; + else if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; + } + else { + s += "\n"; + writeToStderr(s); + } + } + + debug(format("hook reply is '%1%'") % reply); + + if (reply == "decline") + return rpDecline; + else if (reply == "decline-permanently") { + worker.tryBuildHook = false; + worker.hook = 0; + return rpDecline; + } + else if (reply == "postpone") + return rpPostpone; + else if (reply != "accept") + throw Error(format("bad hook reply '%1%'") % reply); + + } catch (SysError & e) { + if (e.errNo == EPIPE) { + printError("build hook died unexpectedly: %s", + chomp(drainFD(worker.hook->fromHook.readSide.get()))); + worker.hook = 0; + return rpDecline; + } else + throw; + } + + hook = std::move(worker.hook); + + machineName = readLine(hook->fromHook.readSide.get()); + + /* Tell the hook all the inputs that have to be copied to the + remote system. */ + hook->sink << inputPaths; + + /* Tell the hooks the missing outputs that have to be copied back + from the remote system. */ + hook->sink << missingPaths; + + hook->sink = FdSink(); + hook->toHook.writeSide = -1; + + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + set fds; + fds.insert(hook->fromHook.readSide.get()); + fds.insert(hook->builderOut.readSide.get()); + worker.childStarted(shared_from_this(), fds, false, false); + + return rpAccept; +} + + +void chmod_(const Path & path, mode_t mode) +{ + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("setting permissions on '%1%'") % path); +} + + +int childEntry(void * arg) +{ + ((DerivationGoal *) arg)->runChild(); + return 1; +} + + +PathSet DerivationGoal::exportReferences(PathSet storePaths) +{ + PathSet paths; + + for (auto storePath : storePaths) { + + /* Check that the store path is valid. */ + if (!worker.store.isInStore(storePath)) + throw BuildError(format("'exportReferencesGraph' contains a non-store path '%1%'") + % storePath); + + storePath = worker.store.toStorePath(storePath); + + if (!inputPaths.count(storePath)) + throw BuildError("cannot export references of path '%s' because it is not in the input closure of the derivation", storePath); + + worker.store.computeFSClosure(storePath, paths); + } + + /* If there are derivations in the graph, then include their + outputs as well. This is useful if you want to do things + like passing all build-time dependencies of some path to a + derivation that builds a NixOS DVD image. */ + PathSet paths2(paths); + + for (auto & j : paths2) { + if (isDerivation(j)) { + Derivation drv = worker.store.derivationFromPath(j); + for (auto & k : drv.outputs) + worker.store.computeFSClosure(k.second.path, paths); + } + } + + return paths; +} + +static std::once_flag dns_resolve_flag; + +static void preloadNSS() { + /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of + one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already + been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to + load its lookup libraries in the parent before any child gets a chance to. */ + std::call_once(dns_resolve_flag, []() { + struct addrinfo *res = NULL; + + if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) { + if (res) freeaddrinfo(res); + } + }); +} + +void DerivationGoal::startBuilder() +{ + /* Right platform? */ + if (!parsedDrv->canBuildLocally()) + throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}", + drv->platform, + concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()), + drvPath, + settings.thisSystem, + concatStringsSep(", ", settings.systemFeatures)); + + if (drv->isBuiltin()) + preloadNSS(); + +#if __APPLE__ + additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); +#endif + + /* Are we doing a chroot build? */ + { + auto noChroot = parsedDrv->getBoolAttr("__noChroot"); + if (settings.sandboxMode == smEnabled) { + if (noChroot) + throw Error(format("derivation '%1%' has '__noChroot' set, " + "but that's not allowed when 'sandbox' is 'true'") % drvPath); +#if __APPLE__ + if (additionalSandboxProfile != "") + throw Error(format("derivation '%1%' specifies a sandbox profile, " + "but this is only allowed when 'sandbox' is 'relaxed'") % drvPath); +#endif + useChroot = true; + } + else if (settings.sandboxMode == smDisabled) + useChroot = false; + else if (settings.sandboxMode == smRelaxed) + useChroot = !fixedOutput && !noChroot; + } + + if (worker.store.storeDir != worker.store.realStoreDir) { + #if __linux__ + useChroot = true; + #else + throw Error("building using a diverted store is not supported on this platform"); + #endif + } + + /* If `build-users-group' is not empty, then we have to build as + one of the members of that group. */ + if (settings.buildUsersGroup != "" && getuid() == 0) { +#if defined(__linux__) || defined(__APPLE__) + buildUser = std::make_unique(); + + /* Make sure that no other processes are executing under this + uid. */ + buildUser->kill(); +#else + /* Don't know how to block the creation of setuid/setgid + binaries on this platform. */ + throw Error("build users are not supported on this platform for security reasons"); +#endif + } + + /* Create a temporary directory where the build will take + place. */ + auto drvName = storePathToName(drvPath); + tmpDir = createTempDir("", "nix-build-" + drvName, false, false, 0700); + + chownToBuilder(tmpDir); + + /* Substitute output placeholders with the actual output paths. */ + for (auto & output : drv->outputs) + inputRewrites[hashPlaceholder(output.first)] = output.second.path; + + /* Construct the environment passed to the builder. */ + initEnv(); + + writeStructuredAttrs(); + + /* Handle exportReferencesGraph(), if set. */ + if (!parsedDrv->getStructuredAttrs()) { + /* The `exportReferencesGraph' feature allows the references graph + to be passed to a builder. This attribute should be a list of + pairs [name1 path1 name2 path2 ...]. The references graph of + each `pathN' will be stored in a text file `nameN' in the + temporary build directory. The text files have the format used + by `nix-store --register-validity'. However, the deriver + fields are left empty. */ + string s = get(drv->env, "exportReferencesGraph"); + Strings ss = tokenizeString(s); + if (ss.size() % 2 != 0) + throw BuildError(format("odd number of tokens in 'exportReferencesGraph': '%1%'") % s); + for (Strings::iterator i = ss.begin(); i != ss.end(); ) { + string fileName = *i++; + checkStoreName(fileName); /* !!! abuse of this function */ + Path storePath = *i++; + + /* Write closure info to . */ + writeFile(tmpDir + "/" + fileName, + worker.store.makeValidityRegistration( + exportReferences({storePath}), false, false)); + } + } + + if (useChroot) { + + /* Allow a user-configurable set of directories from the + host file system. */ + PathSet dirs = settings.sandboxPaths; + PathSet dirs2 = settings.extraSandboxPaths; + dirs.insert(dirs2.begin(), dirs2.end()); + + dirsInChroot.clear(); + + for (auto i : dirs) { + if (i.empty()) continue; + bool optional = false; + if (i[i.size() - 1] == '?') { + optional = true; + i.pop_back(); + } + size_t p = i.find('='); + if (p == string::npos) + dirsInChroot[i] = {i, optional}; + else + dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional}; + } + dirsInChroot[tmpDirInSandbox] = tmpDir; + + /* Add the closure of store paths to the chroot. */ + PathSet closure; + for (auto & i : dirsInChroot) + try { + if (worker.store.isInStore(i.second.source)) + worker.store.computeFSClosure(worker.store.toStorePath(i.second.source), closure); + } catch (InvalidPath & e) { + } catch (Error & e) { + throw Error(format("while processing 'sandbox-paths': %s") % e.what()); + } + for (auto & i : closure) + dirsInChroot[i] = i; + + PathSet allowedPaths = settings.allowedImpureHostPrefixes; + + /* This works like the above, except on a per-derivation level */ + auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings()); + + for (auto & i : impurePaths) { + bool found = false; + /* Note: we're not resolving symlinks here to prevent + giving a non-root user info about inaccessible + files. */ + Path canonI = canonPath(i); + /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */ + for (auto & a : allowedPaths) { + Path canonA = canonPath(a); + if (canonI == canonA || isInDir(canonI, canonA)) { + found = true; + break; + } + } + if (!found) + throw Error(format("derivation '%1%' requested impure path '%2%', but it was not in allowed-impure-host-deps") % drvPath % i); + + dirsInChroot[i] = i; + } + +#if __linux__ + /* Create a temporary directory in which we set up the chroot + environment using bind-mounts. We put it in the Nix store + to ensure that we can create hard-links to non-directory + inputs in the fake Nix store in the chroot (see below). */ + chrootRootDir = worker.store.toRealPath(drvPath) + ".chroot"; + deletePath(chrootRootDir); + + /* Clean up the chroot directory automatically. */ + autoDelChroot = std::make_shared(chrootRootDir); + + printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir); + + if (mkdir(chrootRootDir.c_str(), 0750) == -1) + throw SysError(format("cannot create '%1%'") % chrootRootDir); + + if (buildUser && chown(chrootRootDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % chrootRootDir); + + /* Create a writable /tmp in the chroot. Many builders need + this. (Of course they should really respect $TMPDIR + instead.) */ + Path chrootTmpDir = chrootRootDir + "/tmp"; + createDirs(chrootTmpDir); + chmod_(chrootTmpDir, 01777); + + /* Create a /etc/passwd with entries for the build user and the + nobody account. The latter is kind of a hack to support + Samba-in-QEMU. */ + createDirs(chrootRootDir + "/etc"); + + writeFile(chrootRootDir + "/etc/passwd", fmt( + "root:x:0:0:Nix build user:%3%:/noshell\n" + "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n" + "nobody:x:65534:65534:Nobody:/:/noshell\n", + sandboxUid, sandboxGid, settings.sandboxBuildDir)); + + /* Declare the build user's group so that programs get a consistent + view of the system (e.g., "id -gn"). */ + writeFile(chrootRootDir + "/etc/group", + (format( + "root:x:0:\n" + "nixbld:!:%1%:\n" + "nogroup:x:65534:\n") % sandboxGid).str()); + + /* Create /etc/hosts with localhost entry. */ + if (!fixedOutput) + writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n"); + + /* Make the closure of the inputs available in the chroot, + rather than the whole Nix store. This prevents any access + to undeclared dependencies. Directories are bind-mounted, + while other inputs are hard-linked (since only directories + can be bind-mounted). !!! As an extra security + precaution, make the fake Nix store only writable by the + build user. */ + Path chrootStoreDir = chrootRootDir + worker.store.storeDir; + createDirs(chrootStoreDir); + chmod_(chrootStoreDir, 01775); + + if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % chrootStoreDir); + + for (auto & i : inputPaths) { + Path r = worker.store.toRealPath(i); + struct stat st; + if (lstat(r.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % i); + if (S_ISDIR(st.st_mode)) + dirsInChroot[i] = r; + else { + Path p = chrootRootDir + i; + debug("linking '%1%' to '%2%'", p, r); + if (link(r.c_str(), p.c_str()) == -1) { + /* Hard-linking fails if we exceed the maximum + link count on a file (e.g. 32000 of ext3), + which is quite possible after a `nix-store + --optimise'. */ + if (errno != EMLINK) + throw SysError(format("linking '%1%' to '%2%'") % p % i); + StringSink sink; + dumpPath(r, sink); + StringSource source(*sink.s); + restorePath(p, source); + } + } + } + + /* If we're repairing, checking or rebuilding part of a + multiple-outputs derivation, it's possible that we're + rebuilding a path that is in settings.dirsInChroot + (typically the dependencies of /bin/sh). Throw them + out. */ + for (auto & i : drv->outputs) + dirsInChroot.erase(i.second.path); + +#elif __APPLE__ + /* We don't really have any parent prep work to do (yet?) + All work happens in the child, instead. */ +#else + throw Error("sandboxing builds is not supported on this platform"); +#endif + } + + if (needsHashRewrite()) { + + if (pathExists(homeDir)) + throw Error(format("directory '%1%' exists; please remove it") % homeDir); + + /* We're not doing a chroot build, but we have some valid + output paths. Since we can't just overwrite or delete + them, we have to do hash rewriting: i.e. in the + environment/arguments passed to the build, we replace the + hashes of the valid outputs with unique dummy strings; + after the build, we discard the redirected outputs + corresponding to the valid outputs, and rewrite the + contents of the new outputs to replace the dummy strings + with the actual hashes. */ + if (validPaths.size() > 0) + for (auto & i : validPaths) + addHashRewrite(i); + + /* If we're repairing, then we don't want to delete the + corrupt outputs in advance. So rewrite them as well. */ + if (buildMode == bmRepair) + for (auto & i : missingPaths) + if (worker.store.isValidPath(i) && pathExists(i)) { + addHashRewrite(i); + redirectedBadOutputs.insert(i); + } + } + + if (useChroot && settings.preBuildHook != "" && dynamic_cast(drv.get())) { + printMsg(lvlChatty, format("executing pre-build hook '%1%'") + % settings.preBuildHook); + auto args = useChroot ? Strings({drvPath, chrootRootDir}) : + Strings({ drvPath }); + enum BuildHookState { + stBegin, + stExtraChrootDirs + }; + auto state = stBegin; + auto lines = runProgram(settings.preBuildHook, false, args); + auto lastPos = std::string::size_type{0}; + for (auto nlPos = lines.find('\n'); nlPos != string::npos; + nlPos = lines.find('\n', lastPos)) { + auto line = std::string{lines, lastPos, nlPos - lastPos}; + lastPos = nlPos + 1; + if (state == stBegin) { + if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") { + state = stExtraChrootDirs; + } else { + throw Error(format("unknown pre-build hook command '%1%'") + % line); + } + } else if (state == stExtraChrootDirs) { + if (line == "") { + state = stBegin; + } else { + auto p = line.find('='); + if (p == string::npos) + dirsInChroot[line] = line; + else + dirsInChroot[string(line, 0, p)] = string(line, p + 1); + } + } + } + } + + /* Run the builder. */ + printMsg(lvlChatty, format("executing builder '%1%'") % drv->builder); + + /* Create the log file. */ + Path logFile = openLogFile(); + + /* Create a pipe to get the output of the builder. */ + //builderOut.create(); + + builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY); + if (!builderOut.readSide) + throw SysError("opening pseudoterminal master"); + + std::string slaveName(ptsname(builderOut.readSide.get())); + + if (buildUser) { + if (chmod(slaveName.c_str(), 0600)) + throw SysError("changing mode of pseudoterminal slave"); + + if (chown(slaveName.c_str(), buildUser->getUID(), 0)) + throw SysError("changing owner of pseudoterminal slave"); + } else { + if (grantpt(builderOut.readSide.get())) + throw SysError("granting access to pseudoterminal slave"); + } + + #if 0 + // Mount the pt in the sandbox so that the "tty" command works. + // FIXME: this doesn't work with the new devpts in the sandbox. + if (useChroot) + dirsInChroot[slaveName] = {slaveName, false}; + #endif + + if (unlockpt(builderOut.readSide.get())) + throw SysError("unlocking pseudoterminal"); + + builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY); + if (!builderOut.writeSide) + throw SysError("opening pseudoterminal slave"); + + // Put the pt into raw mode to prevent \n -> \r\n translation. + struct termios term; + if (tcgetattr(builderOut.writeSide.get(), &term)) + throw SysError("getting pseudoterminal attributes"); + + cfmakeraw(&term); + + if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term)) + throw SysError("putting pseudoterminal into raw mode"); + + result.startTime = time(0); + + /* Fork a child to build the package. */ + ProcessOptions options; + +#if __linux__ + if (useChroot) { + /* Set up private namespaces for the build: + + - The PID namespace causes the build to start as PID 1. + Processes outside of the chroot are not visible to those + on the inside, but processes inside the chroot are + visible from the outside (though with different PIDs). + + - The private mount namespace ensures that all the bind + mounts we do will only show up in this process and its + children, and will disappear automatically when we're + done. + + - The private network namespace ensures that the builder + cannot talk to the outside world (or vice versa). It + only has a private loopback interface. (Fixed-output + derivations are not run in a private network namespace + to allow functions like fetchurl to work.) + + - The IPC namespace prevents the builder from communicating + with outside processes using SysV IPC mechanisms (shared + memory, message queues, semaphores). It also ensures + that all IPC objects are destroyed when the builder + exits. + + - The UTS namespace ensures that builders see a hostname of + localhost rather than the actual hostname. + + We use a helper process to do the clone() to work around + clone() being broken in multi-threaded programs due to + at-fork handlers not being run. Note that we use + CLONE_PARENT to ensure that the real builder is parented to + us. + */ + + if (!fixedOutput) + privateNetwork = true; + + userNamespaceSync.create(); + + options.allowVfork = false; + + Pid helper = startProcess([&]() { + + /* Drop additional groups here because we can't do it + after we've created the new user namespace. FIXME: + this means that if we're not root in the parent + namespace, we can't drop additional groups; they will + be mapped to nogroup in the child namespace. There does + not seem to be a workaround for this. (But who can tell + from reading user_namespaces(7)?) + See also https://lwn.net/Articles/621612/. */ + if (getuid() == 0 && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + + size_t stackSize = 1 * 1024 * 1024; + char * stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + int flags = CLONE_NEWUSER | CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) + flags |= CLONE_NEWNET; + + pid_t child = clone(childEntry, stack + stackSize, flags, this); + if (child == -1 && errno == EINVAL) { + /* Fallback for Linux < 2.13 where CLONE_NEWPID and + CLONE_PARENT are not allowed together. */ + flags &= ~CLONE_NEWPID; + child = clone(childEntry, stack + stackSize, flags, this); + } + if (child == -1 && (errno == EPERM || errno == EINVAL)) { + /* Some distros patch Linux to not allow unpriveleged + * user namespaces. If we get EPERM or EINVAL, try + * without CLONE_NEWUSER and see if that works. + */ + flags &= ~CLONE_NEWUSER; + child = clone(childEntry, stack + stackSize, flags, this); + } + /* Otherwise exit with EPERM so we can handle this in the + parent. This is only done when sandbox-fallback is set + to true (the default). */ + if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback) + _exit(1); + if (child == -1) throw SysError("cloning builder process"); + + writeFull(builderOut.writeSide.get(), std::to_string(child) + "\n"); + _exit(0); + }, options); + + int res = helper.wait(); + if (res != 0 && settings.sandboxFallback) { + useChroot = false; + initTmpDir(); + goto fallback; + } else if (res != 0) + throw Error("unable to start build process"); + + userNamespaceSync.readSide = -1; + + pid_t tmp; + if (!string2Int(readLine(builderOut.readSide.get()), tmp)) abort(); + pid = tmp; + + /* Set the UID/GID mapping of the builder's user namespace + such that the sandbox user maps to the build user, or to + the calling user (if build users are disabled). */ + uid_t hostUid = buildUser ? buildUser->getUID() : getuid(); + uid_t hostGid = buildUser ? buildUser->getGID() : getgid(); + + writeFile("/proc/" + std::to_string(pid) + "/uid_map", + (format("%d %d 1") % sandboxUid % hostUid).str()); + + writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny"); + + writeFile("/proc/" + std::to_string(pid) + "/gid_map", + (format("%d %d 1") % sandboxGid % hostGid).str()); + + /* Signal the builder that we've updated its user + namespace. */ + writeFull(userNamespaceSync.writeSide.get(), "1"); + userNamespaceSync.writeSide = -1; + + } else +#endif + { + fallback: + options.allowVfork = !buildUser && !drv->isBuiltin(); + pid = startProcess([&]() { + runChild(); + }, options); + } + + /* parent */ + pid.setSeparatePG(true); + builderOut.writeSide = -1; + worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true); + + /* Check if setting up the build environment failed. */ + while (true) { + string msg = readLine(builderOut.readSide.get()); + if (string(msg, 0, 1) == "\1") { + if (msg.size() == 1) break; + throw Error(string(msg, 1)); + } + debug(msg); + } +} + + +void DerivationGoal::initTmpDir() { + /* In a sandbox, for determinism, always use the same temporary + directory. */ +#if __linux__ + tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir; +#else + tmpDirInSandbox = tmpDir; +#endif + + /* In non-structured mode, add all bindings specified in the + derivation via the environment, except those listed in the + passAsFile attribute. Those are passed as file names pointing + to temporary files containing the contents. Note that + passAsFile is ignored in structure mode because it's not + needed (attributes are not passed through the environment, so + there is no size constraint). */ + if (!parsedDrv->getStructuredAttrs()) { + + StringSet passAsFile = tokenizeString(get(drv->env, "passAsFile")); + int fileNr = 0; + for (auto & i : drv->env) { + if (passAsFile.find(i.first) == passAsFile.end()) { + env[i.first] = i.second; + } else { + string fn = ".attr-" + std::to_string(fileNr++); + Path p = tmpDir + "/" + fn; + writeFile(p, rewriteStrings(i.second, inputRewrites)); + chownToBuilder(p); + env[i.first + "Path"] = tmpDirInSandbox + "/" + fn; + } + } + + } + + /* For convenience, set an environment pointing to the top build + directory. */ + env["NIX_BUILD_TOP"] = tmpDirInSandbox; + + /* Also set TMPDIR and variants to point to this directory. */ + env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox; + + /* Explicitly set PWD to prevent problems with chroot builds. In + particular, dietlibc cannot figure out the cwd because the + inode of the current directory doesn't appear in .. (because + getdents returns the inode of the mount point). */ + env["PWD"] = tmpDirInSandbox; +} + +void DerivationGoal::initEnv() +{ + env.clear(); + + /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when + PATH is not set. We don't want this, so we fill it in with some dummy + value. */ + env["PATH"] = "/path-not-set"; + + /* Set HOME to a non-existing path to prevent certain programs from using + /etc/passwd (or NIS, or whatever) to locate the home directory (for + example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd + if HOME is not set, but they will just assume that the settings file + they are looking for does not exist if HOME is set but points to some + non-existing path. */ + env["HOME"] = homeDir; + + /* Tell the builder where the Nix store is. Usually they + shouldn't care, but this is useful for purity checking (e.g., + the compiler or linker might only want to accept paths to files + in the store or in the build directory). */ + env["NIX_STORE"] = worker.store.storeDir; + + /* The maximum number of cores to utilize for parallel building. */ + env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str(); + + initTmpDir(); + + /* Compatibility hack with Nix <= 0.7: if this is a fixed-output + derivation, tell the builder, so that for instance `fetchurl' + can skip checking the output. On older Nixes, this environment + variable won't be set, so `fetchurl' will do the check. */ + if (fixedOutput) env["NIX_OUTPUT_CHECKED"] = "1"; + + /* *Only* if this is a fixed-output derivation, propagate the + values of the environment variables specified in the + `impureEnvVars' attribute to the builder. This allows for + instance environment variables for proxy configuration such as + `http_proxy' to be easily passed to downloaders like + `fetchurl'. Passing such environment variables from the caller + to the builder is generally impure, but the output of + fixed-output derivations is by definition pure (since we + already know the cryptographic hash of the output). */ + if (fixedOutput) { + for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings())) + env[i] = getEnv(i); + } + + /* Currently structured log messages piggyback on stderr, but we + may change that in the future. So tell the builder which file + descriptor to use for that. */ + env["NIX_LOG_FD"] = "2"; + + /* Trigger colored output in various tools. */ + env["TERM"] = "xterm-256color"; +} + + +static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); + + +void DerivationGoal::writeStructuredAttrs() +{ + auto & structuredAttrs = parsedDrv->getStructuredAttrs(); + if (!structuredAttrs) return; + + auto json = *structuredAttrs; + + /* Add an "outputs" object containing the output paths. */ + nlohmann::json outputs; + for (auto & i : drv->outputs) + outputs[i.first] = rewriteStrings(i.second.path, inputRewrites); + json["outputs"] = outputs; + + /* Handle exportReferencesGraph. */ + auto e = json.find("exportReferencesGraph"); + if (e != json.end() && e->is_object()) { + for (auto i = e->begin(); i != e->end(); ++i) { + std::ostringstream str; + { + JSONPlaceholder jsonRoot(str, true); + PathSet storePaths; + for (auto & p : *i) + storePaths.insert(p.get()); + worker.store.pathInfoToJSON(jsonRoot, + exportReferences(storePaths), false, true); + } + json[i.key()] = nlohmann::json::parse(str.str()); // urgh + } + } + + writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.json"); + + /* As a convenience to bash scripts, write a shell file that + maps all attributes that are representable in bash - + namely, strings, integers, nulls, Booleans, and arrays and + objects consisting entirely of those values. (So nested + arrays or objects are not supported.) */ + + auto handleSimpleType = [](const nlohmann::json & value) -> std::optional { + if (value.is_string()) + return shellEscape(value); + + if (value.is_number()) { + auto f = value.get(); + if (std::ceil(f) == f) + return std::to_string(value.get()); + } + + if (value.is_null()) + return std::string("''"); + + if (value.is_boolean()) + return value.get() ? std::string("1") : std::string(""); + + return {}; + }; + + std::string jsonSh; + + for (auto i = json.begin(); i != json.end(); ++i) { + + if (!std::regex_match(i.key(), shVarName)) continue; + + auto & value = i.value(); + + auto s = handleSimpleType(value); + if (s) + jsonSh += fmt("declare %s=%s\n", i.key(), *s); + + else if (value.is_array()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += *s3; s2 += ' '; + } + + if (good) + jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2); + } + + else if (value.is_object()) { + std::string s2; + bool good = true; + + for (auto i = value.begin(); i != value.end(); ++i) { + auto s3 = handleSimpleType(i.value()); + if (!s3) { good = false; break; } + s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3); + } + + if (good) + jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2); + } + } + + writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites)); + chownToBuilder(tmpDir + "/.attrs.sh"); +} + + +void DerivationGoal::chownToBuilder(const Path & path) +{ + if (!buildUser) return; + if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1) + throw SysError(format("cannot change ownership of '%1%'") % path); +} + + +void setupSeccomp() +{ +#if __linux__ + if (!settings.filterSyscalls) return; +#if HAVE_SECCOMP + scmp_filter_ctx ctx; + + if (!(ctx = seccomp_init(SCMP_ACT_ALLOW))) + throw SysError("unable to initialize seccomp mode 2"); + + Finally cleanup([&]() { + seccomp_release(ctx); + }); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) + throw SysError("unable to add 32-bit seccomp architecture"); + + if (nativeSystem == "x86_64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0) + throw SysError("unable to add X32 seccomp architecture"); + + if (nativeSystem == "aarch64-linux" && + seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0) + printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes"); + + /* Prevent builders from creating setuid/setgid binaries. */ + for (int perm : { S_ISUID, S_ISGID }) { + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1, + SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1, + SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0) + throw SysError("unable to add seccomp rule"); + } + + /* Prevent builders from creating EAs or ACLs. Not all filesystems + support these, and they're not allowed in the Nix store because + they're not representable in the NAR serialisation. */ + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) + throw SysError("unable to add seccomp rule"); + + if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0) + throw SysError("unable to set 'no new privileges' seccomp attribute"); + + if (seccomp_load(ctx) != 0) + throw SysError("unable to load seccomp BPF program"); +#else + throw Error( + "seccomp is not supported on this platform; " + "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!"); +#endif +#endif +} + + +void DerivationGoal::runChild() +{ + /* Warning: in the child we should absolutely not make any SQLite + calls! */ + + try { /* child */ + + commonChildInit(builderOut); + + try { + setupSeccomp(); + } catch (...) { + if (buildUser) throw; + } + + bool setUser = true; + + /* Make the contents of netrc available to builtin:fetchurl + (which may run under a different uid and/or in a sandbox). */ + std::string netrcData; + try { + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") + netrcData = readFile(settings.netrcFile); + } catch (SysError &) { } + +#if __linux__ + if (useChroot) { + + userNamespaceSync.writeSide = -1; + + if (drainFD(userNamespaceSync.readSide.get()) != "1") + throw Error("user namespace initialisation failed"); + + userNamespaceSync.readSide = -1; + + if (privateNetwork) { + + /* Initialise the loopback interface. */ + AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP)); + if (!fd) throw SysError("cannot open IP socket"); + + struct ifreq ifr; + strcpy(ifr.ifr_name, "lo"); + ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING; + if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1) + throw SysError("cannot set loopback interface flags"); + } + + /* Set the hostname etc. to fixed values. */ + char hostname[] = "localhost"; + if (sethostname(hostname, sizeof(hostname)) == -1) + throw SysError("cannot set host name"); + char domainname[] = "(none)"; // kernel default + if (setdomainname(domainname, sizeof(domainname)) == -1) + throw SysError("cannot set domain name"); + + /* Make all filesystems private. This is necessary + because subtrees may have been mounted as "shared" + (MS_SHARED). (Systemd does this, for instance.) Even + though we have a private mount namespace, mounting + filesystems on top of a shared subtree still propagates + outside of the namespace. Making a subtree private is + local to the namespace, though, so setting MS_PRIVATE + does not affect the outside world. */ + if (mount(0, "/", 0, MS_REC|MS_PRIVATE, 0) == -1) { + throw SysError("unable to make '/' private mount"); + } + + /* Bind-mount chroot directory to itself, to treat it as a + different filesystem from /, as needed for pivot_root. */ + if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1) + throw SysError(format("unable to bind mount '%1%'") % chrootRootDir); + + /* Set up a nearly empty /dev, unless the user asked to + bind-mount the host /dev. */ + Strings ss; + if (dirsInChroot.find("/dev") == dirsInChroot.end()) { + createDirs(chrootRootDir + "/dev/shm"); + createDirs(chrootRootDir + "/dev/pts"); + ss.push_back("/dev/full"); + if (settings.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) + ss.push_back("/dev/kvm"); + ss.push_back("/dev/null"); + ss.push_back("/dev/random"); + ss.push_back("/dev/tty"); + ss.push_back("/dev/urandom"); + ss.push_back("/dev/zero"); + createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd"); + createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin"); + createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout"); + createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr"); + } + + /* Fixed-output derivations typically need to access the + network, so give them access to /etc/resolv.conf and so + on. */ + if (fixedOutput) { + ss.push_back("/etc/resolv.conf"); + + // Only use nss functions to resolve hosts and + // services. Don’t use it for anything else that may + // be configured for this system. This limits the + // potential impurities introduced in fixed outputs. + writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n"); + + ss.push_back("/etc/services"); + ss.push_back("/etc/hosts"); + if (pathExists("/var/run/nscd/socket")) + ss.push_back("/var/run/nscd/socket"); + } + + for (auto & i : ss) dirsInChroot.emplace(i, i); + + /* Bind-mount all the directories from the "host" + filesystem that we want in the chroot + environment. */ + auto doBind = [&](const Path & source, const Path & target, bool optional = false) { + debug(format("bind mounting '%1%' to '%2%'") % source % target); + struct stat st; + if (stat(source.c_str(), &st) == -1) { + if (optional && errno == ENOENT) + return; + else + throw SysError("getting attributes of path '%1%'", source); + } + if (S_ISDIR(st.st_mode)) + createDirs(target); + else { + createDirs(dirOf(target)); + writeFile(target, ""); + } + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + for (auto & i : dirsInChroot) { + if (i.second.source == "/proc") continue; // backwards compatibility + doBind(i.second.source, chrootRootDir + i.first, i.second.optional); + } + + /* Bind a new instance of procfs on /proc. */ + createDirs(chrootRootDir + "/proc"); + if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1) + throw SysError("mounting /proc"); + + /* Mount a new tmpfs on /dev/shm to ensure that whatever + the builder puts in /dev/shm is cleaned up automatically. */ + if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0, + fmt("size=%s", settings.sandboxShmSize).c_str()) == -1) + throw SysError("mounting /dev/shm"); + + /* Mount a new devpts on /dev/pts. Note that this + requires the kernel to be compiled with + CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case + if /dev/ptx/ptmx exists). */ + if (pathExists("/dev/pts/ptmx") && + !pathExists(chrootRootDir + "/dev/ptmx") + && !dirsInChroot.count("/dev/pts")) + { + if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0) + { + createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx"); + + /* Make sure /dev/pts/ptmx is world-writable. With some + Linux versions, it is created with permissions 0. */ + chmod_(chrootRootDir + "/dev/pts/ptmx", 0666); + } else { + if (errno != EINVAL) + throw SysError("mounting /dev/pts"); + doBind("/dev/pts", chrootRootDir + "/dev/pts"); + doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx"); + } + } + + /* Do the chroot(). */ + if (chdir(chrootRootDir.c_str()) == -1) + throw SysError(format("cannot change directory to '%1%'") % chrootRootDir); + + if (mkdir("real-root", 0) == -1) + throw SysError("cannot create real-root directory"); + + if (pivot_root(".", "real-root") == -1) + throw SysError(format("cannot pivot old root directory onto '%1%'") % (chrootRootDir + "/real-root")); + + if (chroot(".") == -1) + throw SysError(format("cannot change root directory to '%1%'") % chrootRootDir); + + if (umount2("real-root", MNT_DETACH) == -1) + throw SysError("cannot unmount real root filesystem"); + + if (rmdir("real-root") == -1) + throw SysError("cannot remove real-root directory"); + + /* Switch to the sandbox uid/gid in the user namespace, + which corresponds to the build user or calling user in + the parent namespace. */ + if (setgid(sandboxGid) == -1) + throw SysError("setgid failed"); + if (setuid(sandboxUid) == -1) + throw SysError("setuid failed"); + + setUser = false; + } +#endif + + if (chdir(tmpDirInSandbox.c_str()) == -1) + throw SysError(format("changing into '%1%'") % tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + +#if __linux__ + /* Change the personality to 32-bit if we're doing an + i686-linux build on an x86_64-linux machine. */ + struct utsname utsbuf; + uname(&utsbuf); + if (drv->platform == "i686-linux" && + (settings.thisSystem == "x86_64-linux" || + (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set i686-linux personality"); + } + + /* Impersonate a Linux 2.6 machine to get some determinism in + builds that depend on the kernel version. */ + if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) { + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */); + } + + /* Disable address space randomization for improved + determinism. */ + int cur = personality(0xffffffff); + if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE); +#endif + + /* Disable core dumps by default. */ + struct rlimit limit = { 0, RLIM_INFINITY }; + setrlimit(RLIMIT_CORE, &limit); + + // FIXME: set other limits to deterministic values? + + /* Fill in the environment. */ + Strings envStrs; + for (auto & i : env) + envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites)); + + /* If we are running in `build-users' mode, then switch to the + user we allocated above. Make sure that we drop all root + privileges. Note that above we have closed all file + descriptors except std*, so that's safe. Also note that + setuid() when run as root sets the real, effective and + saved UIDs. */ + if (setUser && buildUser) { + /* Preserve supplementary groups of the build user, to allow + admins to specify groups such as "kvm". */ + if (!buildUser->getSupplementaryGIDs().empty() && + setgroups(buildUser->getSupplementaryGIDs().size(), + buildUser->getSupplementaryGIDs().data()) == -1) + throw SysError("cannot set supplementary groups of build user"); + + if (setgid(buildUser->getGID()) == -1 || + getgid() != buildUser->getGID() || + getegid() != buildUser->getGID()) + throw SysError("setgid failed"); + + if (setuid(buildUser->getUID()) == -1 || + getuid() != buildUser->getUID() || + geteuid() != buildUser->getUID()) + throw SysError("setuid failed"); + } + + /* Fill in the arguments. */ + Strings args; + + const char *builder = "invalid"; + + if (drv->isBuiltin()) { + ; + } +#if __APPLE__ + else if (getEnv("_NIX_TEST_NO_SANDBOX") == "") { + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; + + if (useChroot) { + + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; + + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : dirsInChroot) { + Path cur = i.first; + while (cur.compare("/") != 0) { + cur = dirOf(cur); + ancestry.insert(cur); + } + } + + /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } + + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) + dirsInChroot[i] = i; + + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; + } else { + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += "(import \"sandbox-defaults.sb\")\n"; + + if (fixedOutput) + sandboxProfile += "(import \"sandbox-network.sb\")\n"; + + /* Our rwx outputs */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : missingPaths) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.c_str()).str(); + } + /* Also add redirected outputs to the chroot */ + for (auto & i : redirectedOutputs) { + sandboxProfile += (format("\t(subpath \"%1%\")\n") % i.second.c_str()).str(); + } + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : dirsInChroot) { + if (i.first != i.second.source) + throw Error(format( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin") + % i.first % i.second.source); + + string path = i.first; + struct stat st; + if (lstat(path.c_str(), &st)) { + if (i.second.optional && errno == ENOENT) + continue; + throw SysError(format("getting attributes of path '%1%'") % path); + } + if (S_ISDIR(st.st_mode)) + sandboxProfile += (format("\t(subpath \"%1%\")\n") % path).str(); + else + sandboxProfile += (format("\t(literal \"%1%\")\n") % path).str(); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += (format("\t(literal \"%1%\")\n") % i.c_str()).str(); + } + sandboxProfile += ")\n"; + + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += "(import \"sandbox-minimal.sb\")\n"; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + Path sandboxFile = tmpDir + "/.sandbox.sb"; + + writeFile(sandboxFile, sandboxProfile); + + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + Path globalTmpDir = canonPath(getEnv("TMPDIR", "/tmp"), true); + + /* They don't like trailing slashes on subpath directives */ + if (globalTmpDir.back() == '/') globalTmpDir.pop_back(); + + builder = "/usr/bin/sandbox-exec"; + args.push_back("sandbox-exec"); + args.push_back("-f"); + args.push_back(sandboxFile); + args.push_back("-D"); + args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); + args.push_back("-D"); + args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/"); + if (allowLocalNetworking) { + args.push_back("-D"); + args.push_back(string("_ALLOW_LOCAL_NETWORKING=1")); + } + args.push_back(drv->builder); + } +#endif + else { + builder = drv->builder.c_str(); + string builderBasename = baseNameOf(drv->builder); + args.push_back(builderBasename); + } + + for (auto & i : drv->args) + args.push_back(rewriteStrings(i, inputRewrites)); + + /* Indicate that we managed to set up the build environment. */ + writeFull(STDERR_FILENO, string("\1\n")); + + /* Execute the program. This should not return. */ + if (drv->isBuiltin()) { + try { + logger = makeJSONLogger(*logger); + + BasicDerivation drv2(*drv); + for (auto & e : drv2.env) + e.second = rewriteStrings(e.second, inputRewrites); + + if (drv->builder == "builtin:fetchurl") + builtinFetchurl(drv2, netrcData); + else if (drv->builder == "builtin:buildenv") + builtinBuildenv(drv2); + else + throw Error(format("unsupported builtin function '%1%'") % string(drv->builder, 8)); + _exit(0); + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "error: " + string(e.what()) + "\n"); + _exit(1); + } + } + + execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + + throw SysError(format("executing '%1%'") % drv->builder); + + } catch (std::exception & e) { + writeFull(STDERR_FILENO, "\1while setting up the build environment: " + string(e.what()) + "\n"); + _exit(1); + } +} + + +/* Parse a list of reference specifiers. Each element must either be + a store path, or the symbolic name of the output of the derivation + (such as `out'). */ +PathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv, const Strings & paths) +{ + PathSet result; + for (auto & i : paths) { + if (store.isStorePath(i)) + result.insert(i); + else if (drv.outputs.find(i) != drv.outputs.end()) + result.insert(drv.outputs.find(i)->second.path); + else throw BuildError( + format("derivation contains an illegal reference specifier '%1%'") % i); + } + return result; +} + + +void DerivationGoal::registerOutputs() +{ + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. */ + if (hook) { + bool allValid = true; + for (auto & i : drv->outputs) + if (!worker.store.isValidPath(i.second.path)) allValid = false; + if (allValid) return; + } + + std::map infos; + + /* Set of inodes seen during calls to canonicalisePathMetaData() + for this build's outputs. This needs to be shared between + outputs to allow hard links between outputs. */ + InodesSeen inodesSeen; + + Path checkSuffix = ".check"; + bool keepPreviousRound = settings.keepFailed || settings.runDiffHook; + + std::exception_ptr delayedException; + + /* Check whether the output paths were created, and grep each + output path to determine what other paths it references. Also make all + output paths read-only. */ + for (auto & i : drv->outputs) { + Path path = i.second.path; + if (missingPaths.find(path) == missingPaths.end()) continue; + + ValidPathInfo info; + + Path actualPath = path; + if (useChroot) { + actualPath = chrootRootDir + path; + if (pathExists(actualPath)) { + /* Move output paths from the chroot to the Nix store. */ + if (buildMode == bmRepair) + replaceValidPath(path, actualPath); + else + if (buildMode != bmCheck && rename(actualPath.c_str(), worker.store.toRealPath(path).c_str()) == -1) + throw SysError(format("moving build output '%1%' from the sandbox to the Nix store") % path); + } + if (buildMode != bmCheck) actualPath = worker.store.toRealPath(path); + } + + if (needsHashRewrite()) { + Path redirected = redirectedOutputs[path]; + if (buildMode == bmRepair + && redirectedBadOutputs.find(path) != redirectedBadOutputs.end() + && pathExists(redirected)) + replaceValidPath(path, redirected); + if (buildMode == bmCheck && redirected != "") + actualPath = redirected; + } + + struct stat st; + if (lstat(actualPath.c_str(), &st) == -1) { + if (errno == ENOENT) + throw BuildError( + format("builder for '%1%' failed to produce output path '%2%'") + % drvPath % path); + throw SysError(format("getting attributes of path '%1%'") % actualPath); + } + +#ifndef __CYGWIN__ + /* Check that the output is not group or world writable, as + that means that someone else can have interfered with the + build. Also, the output should be owned by the build + user. */ + if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || + (buildUser && st.st_uid != buildUser->getUID())) + throw BuildError(format("suspicious ownership or permission on '%1%'; rejecting this build output") % path); +#endif + + /* Apply hash rewriting if necessary. */ + bool rewritten = false; + if (!outputRewrites.empty()) { + printError(format("warning: rewriting hashes in '%1%'; cross fingers") % path); + + /* Canonicalise first. This ensures that the path we're + rewriting doesn't contain a hard link to /etc/shadow or + something like that. */ + canonicalisePathMetaData(actualPath, buildUser ? buildUser->getUID() : -1, inodesSeen); + + /* FIXME: this is in-memory. */ + StringSink sink; + dumpPath(actualPath, sink); + deletePath(actualPath); + sink.s = make_ref(rewriteStrings(*sink.s, outputRewrites)); + StringSource source(*sink.s); + restorePath(actualPath, source); + + rewritten = true; + } + + /* Check that fixed-output derivations produced the right + outputs (i.e., the content hash should match the specified + hash). */ + if (fixedOutput) { + + bool recursive; Hash h; + i.second.parseHashInfo(recursive, h); + + if (!recursive) { + /* The output path should be a regular file without + execute permission. */ + if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0) + throw BuildError( + format("output path '%1%' should be a non-executable regular file") % path); + } + + /* Check the hash. In hash mode, move the path produced by + the derivation to its content-addressed location. */ + Hash h2 = recursive ? hashPath(h.type, actualPath).first : hashFile(h.type, actualPath); + + Path dest = worker.store.makeFixedOutputPath(recursive, h2, storePathToName(path)); + + if (h != h2) { + + /* Throw an error after registering the path as + valid. */ + worker.hashMismatch = true; + delayedException = std::make_exception_ptr( + BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s", + dest, h.to_string(), h2.to_string())); + + Path actualDest = worker.store.toRealPath(dest); + + if (worker.store.isValidPath(dest)) + std::rethrow_exception(delayedException); + + if (actualPath != actualDest) { + PathLocks outputLocks({actualDest}); + deletePath(actualDest); + if (rename(actualPath.c_str(), actualDest.c_str()) == -1) + throw SysError(format("moving '%1%' to '%2%'") % actualPath % dest); + } + + path = dest; + actualPath = actualDest; + } + else + assert(path == dest); + + info.ca = makeFixedOutputCA(recursive, h2); + } + + /* Get rid of all weird permissions. This also checks that + all files are owned by the build user, if applicable. */ + canonicalisePathMetaData(actualPath, + buildUser && !rewritten ? buildUser->getUID() : -1, inodesSeen); + + /* For this output path, find the references to other paths + contained in it. Compute the SHA-256 NAR hash at the same + time. The hash is stored in the database so that we can + verify later on whether nobody has messed with the store. */ + debug("scanning for references inside '%1%'", path); + HashResult hash; + PathSet references = scanForReferences(actualPath, allPaths, hash); + + if (buildMode == bmCheck) { + if (!worker.store.isValidPath(path)) continue; + auto info = *worker.store.queryPathInfo(path); + if (hash.first != info.narHash) { + worker.checkMismatch = true; + if (settings.runDiffHook || settings.keepFailed) { + Path dst = worker.store.toRealPath(path + checkSuffix); + deletePath(dst); + if (rename(actualPath.c_str(), dst.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + path, dst, drvPath, tmpDir); + + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs from '%3%'") + % drvPath % path % dst); + } else + throw NotDeterministic(format("derivation '%1%' may not be deterministic: output '%2%' differs") + % drvPath % path); + } + + /* Since we verified the build, it's now ultimately + trusted. */ + if (!info.ultimate) { + info.ultimate = true; + worker.store.signPathInfo(info); + worker.store.registerValidPaths({info}); + } + + continue; + } + + /* For debugging, print out the referenced and unreferenced + paths. */ + for (auto & i : inputPaths) { + PathSet::iterator j = references.find(i); + if (j == references.end()) + debug(format("unreferenced input: '%1%'") % i); + else + debug(format("referenced input: '%1%'") % i); + } + + if (curRound == nrRounds) { + worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences() + worker.markContentsGood(path); + } + + info.path = path; + info.narHash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = drvPath; + info.ultimate = true; + worker.store.signPathInfo(info); + + if (!info.references.empty()) info.ca.clear(); + + infos[i.first] = info; + } + + if (buildMode == bmCheck) return; + + /* Apply output checks. */ + checkOutputs(infos); + + /* Compare the result with the previous round, and report which + path is different, if any.*/ + if (curRound > 1 && prevInfos != infos) { + assert(prevInfos.size() == infos.size()); + for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j) + if (!(*i == *j)) { + result.isNonDeterministic = true; + Path prev = i->second.path + checkSuffix; + bool prevExists = keepPreviousRound && pathExists(prev); + auto msg = prevExists + ? fmt("output '%1%' of '%2%' differs from '%3%' from previous round", i->second.path, drvPath, prev) + : fmt("output '%1%' of '%2%' differs from previous round", i->second.path, drvPath); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + prev, i->second.path, drvPath, tmpDir); + + if (settings.enforceDeterminism) + throw NotDeterministic(msg); + + printError(msg); + curRound = nrRounds; // we know enough, bail out early + } + } + + /* If this is the first round of several, then move the output out + of the way. */ + if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + Path dst = i.second.path + checkSuffix; + if (rename(i.second.path.c_str(), dst.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % i.second.path % dst); + } + } + + if (curRound < nrRounds) { + prevInfos = infos; + return; + } + + /* Remove the .check directories if we're done. FIXME: keep them + if the result was not determistic? */ + if (curRound == nrRounds) { + for (auto & i : drv->outputs) { + Path prev = i.second.path + checkSuffix; + deletePath(prev); + } + } + + /* Register each output path as valid, and register the sets of + paths referenced by each of them. If there are cycles in the + outputs, this will fail. */ + { + ValidPathInfos infos2; + for (auto & i : infos) infos2.push_back(i.second); + worker.store.registerValidPaths(infos2); + } + + /* In case of a fixed-output derivation hash mismatch, throw an + exception now that we have registered the output as valid. */ + if (delayedException) + std::rethrow_exception(delayedException); +} + + +void DerivationGoal::checkOutputs(const std::map & outputs) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(output.second.path, output.second); + + for (auto & output : outputs) { + auto & outputName = output.first; + auto & info = output.second; + + struct Checks + { + bool ignoreSelfRefs = false; + std::optional maxSize, maxClosureSize; + std::optional allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites; + }; + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const Path & path) + { + uint64_t closureSize = 0; + PathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) continue; + + auto i = outputsByPath.find(path); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = worker.store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(pathsDone, closureSize); + }; + + auto applyChecks = [&](const Checks & checks) + { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes", + info.path, info.narSize, *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes", + info.path, closureSize, *checks.maxClosureSize); + } + + auto checkRefs = [&](const std::optional & value, bool allowed, bool recursive) + { + if (!value) return; + + PathSet spec = parseReferenceSpecifiers(worker.store, *drv, *value); + + PathSet used = recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + PathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += i; + } + throw BuildError("output '%s' is not allowed to refer to the following paths:%s", info.path, badPathsStr); + } + }; + + checkRefs(checks.allowedReferences, true, false); + checkRefs(checks.allowedRequisites, true, true); + checkRefs(checks.disallowedReferences, false, false); + checkRefs(checks.disallowedRequisites, false, true); + }; + + if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) { + auto outputChecks = structuredAttrs->find("outputChecks"); + if (outputChecks != structuredAttrs->end()) { + auto output = outputChecks->find(outputName); + + if (output != outputChecks->end()) { + Checks checks; + + auto maxSize = output->find("maxSize"); + if (maxSize != output->end()) + checks.maxSize = maxSize->get(); + + auto maxClosureSize = output->find("maxClosureSize"); + if (maxClosureSize != output->end()) + checks.maxClosureSize = maxClosureSize->get(); + + auto get = [&](const std::string & name) -> std::optional { + auto i = output->find(name); + if (i != output->end()) { + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get()); + } + checks.disallowedRequisites = res; + return res; + } + return {}; + }; + + checks.allowedReferences = get("allowedReferences"); + checks.allowedRequisites = get("allowedRequisites"); + checks.disallowedReferences = get("disallowedReferences"); + checks.disallowedRequisites = get("disallowedRequisites"); + + applyChecks(checks); + } + } + } else { + // legacy non-structured-attributes case + Checks checks; + checks.ignoreSelfRefs = true; + checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences"); + checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites"); + checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences"); + checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites"); + applyChecks(checks); + } + } +} + + +Path DerivationGoal::openLogFile() +{ + logSize = 0; + + if (!settings.keepLog) return ""; + + string baseName = baseNameOf(drvPath); + + /* Create a log file. */ + Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2)); + createDirs(dir); + + Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2), + settings.compressLog ? ".bz2" : ""); + + fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + if (!fdLogFile) throw SysError(format("creating log file '%1%'") % logFileName); + + logFileSink = std::make_shared(fdLogFile.get()); + + if (settings.compressLog) + logSink = std::shared_ptr(makeCompressionSink("bzip2", *logFileSink)); + else + logSink = logFileSink; + + return logFileName; +} + + +void DerivationGoal::closeLogFile() +{ + auto logSink2 = std::dynamic_pointer_cast(logSink); + if (logSink2) logSink2->finish(); + if (logFileSink) logFileSink->flush(); + logSink = logFileSink = 0; + fdLogFile = -1; +} + + +void DerivationGoal::deleteTmpDir(bool force) +{ + if (tmpDir != "") { + /* Don't keep temporary directories for builtins because they + might have privileged stuff (like a copy of netrc). */ + if (settings.keepFailed && !force && !drv->isBuiltin()) { + printError( + format("note: keeping build directory '%2%'") + % drvPath % tmpDir); + chmod(tmpDir.c_str(), 0755); + } + else + deletePath(tmpDir); + tmpDir = ""; + } +} + + +void DerivationGoal::handleChildOutput(int fd, const string & data) +{ + if ((hook && fd == hook->builderOut.readSide.get()) || + (!hook && fd == builderOut.readSide.get())) + { + logSize += data.size(); + if (settings.maxLogSize && logSize > settings.maxLogSize) { + printError( + format("%1% killed after writing more than %2% bytes of log output") + % getName() % settings.maxLogSize); + killChild(); + done(BuildResult::LogLimitExceeded); + return; + } + + for (auto c : data) + if (c == '\r') + currentLogLinePos = 0; + else if (c == '\n') + flushLine(); + else { + if (currentLogLinePos >= currentLogLine.size()) + currentLogLine.resize(currentLogLinePos + 1); + currentLogLine[currentLogLinePos++] = c; + } + + if (logSink) (*logSink)(data); + } + + if (hook && fd == hook->fromHook.readSide.get()) { + for (auto c : data) + if (c == '\n') { + handleJSONLogMessage(currentHookLine, worker.act, hook->activities, true); + currentHookLine.clear(); + } else + currentHookLine += c; + } +} + + +void DerivationGoal::handleEOF(int fd) +{ + if (!currentLogLine.empty()) flushLine(); + worker.wakeUp(shared_from_this()); +} + + +void DerivationGoal::flushLine() +{ + if (handleJSONLogMessage(currentLogLine, *act, builderActivities, false)) + ; + + else { + if (settings.verboseBuild && + (settings.printRepeatedBuilds || curRound == 1)) + printError(currentLogLine); + else { + logTail.push_back(currentLogLine); + if (logTail.size() > settings.logLines) logTail.pop_front(); + } + + act->result(resBuildLogLine, currentLogLine); + } + + currentLogLine = ""; + currentLogLinePos = 0; +} + + +PathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash) +{ + PathSet result; + for (auto & i : drv->outputs) { + if (!wantOutput(i.first, wantedOutputs)) continue; + bool good = + worker.store.isValidPath(i.second.path) && + (!checkHash || worker.pathContentsGood(i.second.path)); + if (good == returnValid) result.insert(i.second.path); + } + return result; +} + + +Path DerivationGoal::addHashRewrite(const Path & path) +{ + string h1 = string(path, worker.store.storeDir.size() + 1, 32); + string h2 = string(hashString(htSHA256, "rewrite:" + drvPath + ":" + path).to_string(Base32, false), 0, 32); + Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33); + deletePath(p); + assert(path.size() == p.size()); + inputRewrites[h1] = h2; + outputRewrites[h2] = h1; + redirectedOutputs[path] = p; + return p; +} + + +void DerivationGoal::done(BuildResult::Status status, const string & msg) +{ + result.status = status; + result.errorMsg = msg; + amDone(result.success() ? ecSuccess : ecFailed); + if (result.status == BuildResult::TimedOut) + worker.timedOut = true; + if (result.status == BuildResult::PermanentFailure) + worker.permanentFailure = true; + + mcExpectedBuilds.reset(); + mcRunningBuilds.reset(); + + if (result.success()) { + if (status == BuildResult::Built) + worker.doneBuilds++; + } else { + if (status != BuildResult::DependencyFailed) + worker.failedBuilds++; + } + + worker.updateProgress(); +} + + +////////////////////////////////////////////////////////////////////// + + +class SubstitutionGoal : public Goal +{ + friend class Worker; + +private: + /* The store path that should be realised through a substitute. */ + Path storePath; + + /* The remaining substituters. */ + std::list> subs; + + /* The current substituter. */ + std::shared_ptr sub; + + /* Whether a substituter failed. */ + bool substituterFailed = false; + + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr info; + + /* Pipe for the substituter's standard output. */ + Pipe outPipe; + + /* The substituter thread. */ + std::thread thr; + + std::promise promise; + + /* Whether to try to repair a valid path. */ + RepairFlag repair; + + /* Location where we're downloading the substitute. Differs from + storePath when doing a repair. */ + Path destPath; + + std::unique_ptr> maintainExpectedSubstitutions, + maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; + + typedef void (SubstitutionGoal::*GoalState)(); + GoalState state; + +public: + SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair = NoRepair); + ~SubstitutionGoal(); + + void timedOut() override { abort(); }; + + string key() override + { + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + storePathToName(storePath) + "$" + storePath; + } + + void work() override; + + /* The states. */ + void init(); + void tryNext(); + void gotInfo(); + void referencesValid(); + void tryToRun(); + void finished(); + + /* Callback used by the worker to write to the log. */ + void handleChildOutput(int fd, const string & data) override; + void handleEOF(int fd) override; + + Path getStorePath() { return storePath; } + + void amDone(ExitCode result) override + { + Goal::amDone(result); + } +}; + + +SubstitutionGoal::SubstitutionGoal(const Path & storePath, Worker & worker, RepairFlag repair) + : Goal(worker) + , repair(repair) +{ + this->storePath = storePath; + state = &SubstitutionGoal::init; + name = (format("substitution of '%1%'") % storePath).str(); + trace("created"); + maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); +} + + +SubstitutionGoal::~SubstitutionGoal() +{ + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); + } + } catch (...) { + ignoreException(); + } +} + + +void SubstitutionGoal::work() +{ + (this->*state)(); +} + + +void SubstitutionGoal::init() +{ + trace("init"); + + worker.store.addTempRoot(storePath); + + /* If the path already exists we're done. */ + if (!repair && worker.store.isValidPath(storePath)) { + amDone(ecSuccess); + return; + } + + if (settings.readOnlyMode) + throw Error(format("cannot substitute path '%1%' - no write access to the Nix store") % storePath); + + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + + tryNext(); +} + + +void SubstitutionGoal::tryNext() +{ + trace("trying next substituter"); + + if (subs.size() == 0) { + /* None left. Terminate this goal and let someone else deal + with it. */ + debug(format("path '%1%' is required, but there is no substituter that can build it") % storePath); + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + amDone(substituterFailed ? ecFailed : ecNoSubstituters); + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + return; + } + + sub = subs.front(); + subs.pop_front(); + + if (sub->storeDir != worker.store.storeDir) { + tryNext(); + return; + } + + try { + // FIXME: make async + info = sub->queryPathInfo(storePath); + } catch (InvalidPath &) { + tryNext(); + return; + } catch (SubstituterDisabled &) { + if (settings.tryFallback) { + tryNext(); + return; + } + throw; + } catch (Error & e) { + if (settings.tryFallback) { + printError(e.what()); + tryNext(); + return; + } + throw; + } + + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); + + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + worker.updateProgress(); + + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (worker.store.requireSigs + && !sub->isTrusted + && !info->checkSignatures(worker.store, worker.store.getPublicKeys())) + { + printError("warning: substituter '%s' does not have a valid signature for path '%s'", + sub->getUri(), storePath); + tryNext(); + return; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makeSubstitutionGoal(i)); + + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + referencesValid(); + else + state = &SubstitutionGoal::referencesValid; +} + + +void SubstitutionGoal::referencesValid() +{ + trace("all references realised"); + + if (nrFailed > 0) { + debug(format("some references of path '%1%' could not be realised") % storePath); + amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + return; + } + + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); + + state = &SubstitutionGoal::tryToRun; + worker.wakeUp(shared_from_this()); +} + + +void SubstitutionGoal::tryToRun() +{ + trace("trying to run"); + + /* Make sure that we are allowed to start a build. Note that even + if maxBuildJobs == 0 (no local builds allowed), we still allow + a substituter to run. This is because substitutions cannot be + distributed to another machine via the build hook. */ + if (worker.getNrLocalBuilds() >= std::max(1U, (unsigned int) settings.maxBuildJobs)) { + worker.waitForBuildSlot(shared_from_this()); + return; + } + + maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + worker.updateProgress(); + + outPipe.create(); + + promise = std::promise(); + + thr = std::thread([this]() { + try { + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide = -1; }); + + Activity act(*logger, actSubstitute, Logger::Fields{storePath, sub->getUri()}); + PushActivity pact(act.id); + + copyStorePath(ref(sub), ref(worker.store.shared_from_this()), + storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + + state = &SubstitutionGoal::finished; +} + + +void SubstitutionGoal::finished() +{ + trace("substitute finished"); + + thr.join(); + worker.childTerminated(this); + + try { + promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } + + /* Try the next substitute. */ + state = &SubstitutionGoal::tryNext; + worker.wakeUp(shared_from_this()); + return; + } + + worker.markContentsGood(storePath); + + printMsg(lvlChatty, + format("substitution of path '%1%' succeeded") % storePath); + + maintainRunningSubstitutions.reset(); + + maintainExpectedSubstitutions.reset(); + worker.doneSubstitutions++; + + if (maintainExpectedDownload) { + auto fileSize = maintainExpectedDownload->delta; + maintainExpectedDownload.reset(); + worker.doneDownloadSize += fileSize; + } + + worker.doneNarSize += maintainExpectedNar->delta; + maintainExpectedNar.reset(); + + worker.updateProgress(); + + amDone(ecSuccess); +} + + +void SubstitutionGoal::handleChildOutput(int fd, const string & data) +{ +} + + +void SubstitutionGoal::handleEOF(int fd) +{ + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +} + + +////////////////////////////////////////////////////////////////////// + + +static bool working = false; + + +Worker::Worker(LocalStore & store) + : act(*logger, actRealise) + , actDerivations(*logger, actBuilds) + , actSubstitutions(*logger, actCopyPaths) + , store(store) +{ + /* Debugging: prevent recursive workers. */ + if (working) abort(); + working = true; + nrLocalBuilds = 0; + lastWokenUp = steady_time_point::min(); + permanentFailure = false; + timedOut = false; + hashMismatch = false; + checkMismatch = false; +} + + +Worker::~Worker() +{ + working = false; + + /* Explicitly get rid of all strong pointers now. After this all + goals that refer to this worker should be gone. (Otherwise we + are in trouble, since goals may call childTerminated() etc. in + their destructors). */ + topGoals.clear(); + + assert(expectedSubstitutions == 0); + assert(expectedDownloadSize == 0); + assert(expectedNarSize == 0); +} + + +GoalPtr Worker::makeDerivationGoal(const Path & path, + const StringSet & wantedOutputs, BuildMode buildMode) +{ + GoalPtr goal = derivationGoals[path].lock(); + if (!goal) { + goal = std::make_shared(path, wantedOutputs, *this, buildMode); + derivationGoals[path] = goal; + wakeUp(goal); + } else + (dynamic_cast(goal.get()))->addWantedOutputs(wantedOutputs); + return goal; +} + + +std::shared_ptr Worker::makeBasicDerivationGoal(const Path & drvPath, + const BasicDerivation & drv, BuildMode buildMode) +{ + auto goal = std::make_shared(drvPath, drv, *this, buildMode); + wakeUp(goal); + return goal; +} + + +GoalPtr Worker::makeSubstitutionGoal(const Path & path, RepairFlag repair) +{ + GoalPtr goal = substitutionGoals[path].lock(); + if (!goal) { + goal = std::make_shared(path, *this, repair); + substitutionGoals[path] = goal; + wakeUp(goal); + } + return goal; +} + + +static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap) +{ + /* !!! inefficient */ + for (WeakGoalMap::iterator i = goalMap.begin(); + i != goalMap.end(); ) + if (i->second.lock() == goal) { + WeakGoalMap::iterator j = i; ++j; + goalMap.erase(i); + i = j; + } + else ++i; +} + + +void Worker::removeGoal(GoalPtr goal) +{ + nix::removeGoal(goal, derivationGoals); + nix::removeGoal(goal, substitutionGoals); + if (topGoals.find(goal) != topGoals.end()) { + topGoals.erase(goal); + /* If a top-level goal failed, then kill all other goals + (unless keepGoing was set). */ + if (goal->getExitCode() == Goal::ecFailed && !settings.keepGoing) + topGoals.clear(); + } + + /* Wake up goals waiting for any goal to finish. */ + for (auto & i : waitingForAnyGoal) { + GoalPtr goal = i.lock(); + if (goal) wakeUp(goal); + } + + waitingForAnyGoal.clear(); +} + + +void Worker::wakeUp(GoalPtr goal) +{ + goal->trace("woken up"); + addToWeakGoals(awake, goal); +} + + +unsigned Worker::getNrLocalBuilds() +{ + return nrLocalBuilds; +} + + +void Worker::childStarted(GoalPtr goal, const set & fds, + bool inBuildSlot, bool respectTimeouts) +{ + Child child; + child.goal = goal; + child.goal2 = goal.get(); + child.fds = fds; + child.timeStarted = child.lastOutput = steady_time_point::clock::now(); + child.inBuildSlot = inBuildSlot; + child.respectTimeouts = respectTimeouts; + children.emplace_back(child); + if (inBuildSlot) nrLocalBuilds++; +} + + +void Worker::childTerminated(Goal * goal, bool wakeSleepers) +{ + auto i = std::find_if(children.begin(), children.end(), + [&](const Child & child) { return child.goal2 == goal; }); + if (i == children.end()) return; + + if (i->inBuildSlot) { + assert(nrLocalBuilds > 0); + nrLocalBuilds--; + } + + children.erase(i); + + if (wakeSleepers) { + + /* Wake up goals waiting for a build slot. */ + for (auto & j : wantingToBuild) { + GoalPtr goal = j.lock(); + if (goal) wakeUp(goal); + } + + wantingToBuild.clear(); + } +} + + +void Worker::waitForBuildSlot(GoalPtr goal) +{ + debug("wait for build slot"); + if (getNrLocalBuilds() < settings.maxBuildJobs) + wakeUp(goal); /* we can do it right away */ + else + addToWeakGoals(wantingToBuild, goal); +} + + +void Worker::waitForAnyGoal(GoalPtr goal) +{ + debug("wait for any goal"); + addToWeakGoals(waitingForAnyGoal, goal); +} + + +void Worker::waitForAWhile(GoalPtr goal) +{ + debug("wait for a while"); + addToWeakGoals(waitingForAWhile, goal); +} + + +void Worker::run(const Goals & _topGoals) +{ + for (auto & i : _topGoals) topGoals.insert(i); + + debug("entered goal loop"); + + while (1) { + + checkInterrupt(); + + store.autoGC(false); + + /* Call every wake goal (in the ordering established by + CompareGoalPtrs). */ + while (!awake.empty() && !topGoals.empty()) { + Goals awake2; + for (auto & i : awake) { + GoalPtr goal = i.lock(); + if (goal) awake2.insert(goal); + } + awake.clear(); + for (auto & goal : awake2) { + checkInterrupt(); + goal->work(); + if (topGoals.empty()) break; // stuff may have been cancelled + } + } + + if (topGoals.empty()) break; + + /* Wait for input. */ + if (!children.empty() || !waitingForAWhile.empty()) + waitForInput(); + else { + if (awake.empty() && 0 == settings.maxBuildJobs) throw Error( + "unable to start any build; either increase '--max-jobs' " + "or enable remote builds"); + assert(!awake.empty()); + } + } + + /* If --keep-going is not set, it's possible that the main goal + exited while some of its subgoals were still active. But if + --keep-going *is* set, then they must all be finished now. */ + assert(!settings.keepGoing || awake.empty()); + assert(!settings.keepGoing || wantingToBuild.empty()); + assert(!settings.keepGoing || children.empty()); +} + + +void Worker::waitForInput() +{ + printMsg(lvlVomit, "waiting for children"); + + /* Process output from the file descriptors attached to the + children, namely log output and output path creation commands. + We also use this to detect child termination: if we get EOF on + the logger pipe of a build, we assume that the builder has + terminated. */ + + bool useTimeout = false; + struct timeval timeout; + timeout.tv_usec = 0; + auto before = steady_time_point::clock::now(); + + /* If we're monitoring for silence on stdout/stderr, or if there + is a build timeout, then wait for input until the first + deadline for any child. */ + auto nearest = steady_time_point::max(); // nearest deadline + if (settings.minFree.get() != 0) + // Periodicallty wake up to see if we need to run the garbage collector. + nearest = before + std::chrono::seconds(10); + for (auto & i : children) { + if (!i.respectTimeouts) continue; + if (0 != settings.maxSilentTime) + nearest = std::min(nearest, i.lastOutput + std::chrono::seconds(settings.maxSilentTime)); + if (0 != settings.buildTimeout) + nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout)); + } + if (nearest != steady_time_point::max()) { + timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast(nearest - before).count()); + useTimeout = true; + } + + /* If we are polling goals that are waiting for a lock, then wake + up after a few seconds at most. */ + if (!waitingForAWhile.empty()) { + useTimeout = true; + if (lastWokenUp == steady_time_point::min()) + printError("waiting for locks or build slots..."); + if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before; + timeout.tv_sec = std::max(1L, + (long) std::chrono::duration_cast( + lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count()); + } else lastWokenUp = steady_time_point::min(); + + if (useTimeout) + vomit("sleeping %d seconds", timeout.tv_sec); + + /* Use select() to wait for the input side of any logger pipe to + become `available'. Note that `available' (i.e., non-blocking) + includes EOF. */ + fd_set fds; + FD_ZERO(&fds); + int fdMax = 0; + for (auto & i : children) { + for (auto & j : i.fds) { + if (j >= FD_SETSIZE) + throw Error("reached FD_SETSIZE limit"); + FD_SET(j, &fds); + if (j >= fdMax) fdMax = j + 1; + } + } + + if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) { + if (errno == EINTR) return; + throw SysError("waiting for input"); + } + + auto after = steady_time_point::clock::now(); + + /* Process all available file descriptors. FIXME: this is + O(children * fds). */ + decltype(children)::iterator i; + for (auto j = children.begin(); j != children.end(); j = i) { + i = std::next(j); + + checkInterrupt(); + + GoalPtr goal = j->goal.lock(); + assert(goal); + + set fds2(j->fds); + std::vector buffer(4096); + for (auto & k : fds2) { + if (FD_ISSET(k, &fds)) { + ssize_t rd = read(k, buffer.data(), buffer.size()); + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { + debug(format("%1%: got EOF") % goal->getName()); + goal->handleEOF(k); + j->fds.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("%s: read failed", goal->getName()); + } else { + printMsg(lvlVomit, format("%1%: read %2% bytes") + % goal->getName() % rd); + string data((char *) buffer.data(), rd); + j->lastOutput = after; + goal->handleChildOutput(k, data); + } + } + } + + if (goal->getExitCode() == Goal::ecBusy && + 0 != settings.maxSilentTime && + j->respectTimeouts && + after - j->lastOutput >= std::chrono::seconds(settings.maxSilentTime)) + { + printError( + format("%1% timed out after %2% seconds of silence") + % goal->getName() % settings.maxSilentTime); + goal->timedOut(); + } + + else if (goal->getExitCode() == Goal::ecBusy && + 0 != settings.buildTimeout && + j->respectTimeouts && + after - j->timeStarted >= std::chrono::seconds(settings.buildTimeout)) + { + printError( + format("%1% timed out after %2% seconds") + % goal->getName() % settings.buildTimeout); + goal->timedOut(); + } + } + + if (!waitingForAWhile.empty() && lastWokenUp + std::chrono::seconds(settings.pollInterval) <= after) { + lastWokenUp = after; + for (auto & i : waitingForAWhile) { + GoalPtr goal = i.lock(); + if (goal) wakeUp(goal); + } + waitingForAWhile.clear(); + } +} + + +unsigned int Worker::exitStatus() +{ + /* + * 1100100 + * ^^^^ + * |||`- timeout + * ||`-- output hash mismatch + * |`--- build failure + * `---- not deterministic + */ + unsigned int mask = 0; + bool buildFailure = permanentFailure || timedOut || hashMismatch; + if (buildFailure) + mask |= 0x04; // 100 + if (timedOut) + mask |= 0x01; // 101 + if (hashMismatch) + mask |= 0x02; // 102 + if (checkMismatch) { + mask |= 0x08; // 104 + } + + if (mask) + mask |= 0x60; + return mask ? mask : 1; +} + + +bool Worker::pathContentsGood(const Path & path) +{ + std::map::iterator i = pathContentsGoodCache.find(path); + if (i != pathContentsGoodCache.end()) return i->second; + printInfo(format("checking path '%1%'...") % path); + auto info = store.queryPathInfo(path); + bool res; + if (!pathExists(path)) + res = false; + else { + HashResult current = hashPath(info->narHash.type, path); + Hash nullHash(htSHA256); + res = info->narHash == nullHash || info->narHash == current.first; + } + pathContentsGoodCache[path] = res; + if (!res) printError(format("path '%1%' is corrupted or missing!") % path); + return res; +} + + +void Worker::markContentsGood(const Path & path) +{ + pathContentsGoodCache[path] = true; +} + + +////////////////////////////////////////////////////////////////////// + + +static void primeCache(Store & store, const PathSet & paths) +{ + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty()) + throw Error( + "%d derivations need to be built, but neither local builds ('--max-jobs') " + "nor remote builds ('--builders') are enabled", willBuild.size()); +} + + +void LocalStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +{ + Worker worker(*this); + + primeCache(*this, drvPaths); + + Goals goals; + for (auto & i : drvPaths) { + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(i); + if (isDerivation(i2.first)) + goals.insert(worker.makeDerivationGoal(i2.first, i2.second, buildMode)); + else + goals.insert(worker.makeSubstitutionGoal(i, buildMode == bmRepair ? Repair : NoRepair)); + } + + worker.run(goals); + + PathSet failed; + for (auto & i : goals) { + if (i->getExitCode() != Goal::ecSuccess) { + DerivationGoal * i2 = dynamic_cast(i.get()); + if (i2) failed.insert(i2->getDrvPath()); + else failed.insert(dynamic_cast(i.get())->getStorePath()); + } + } + + if (!failed.empty()) + throw Error(worker.exitStatus(), "build of %s failed", showPaths(failed)); +} + + +BuildResult LocalStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + Worker worker(*this); + auto goal = worker.makeBasicDerivationGoal(drvPath, drv, buildMode); + + BuildResult result; + + try { + worker.run(Goals{goal}); + result = goal->getResult(); + } catch (Error & e) { + result.status = BuildResult::MiscFailure; + result.errorMsg = e.msg(); + } + + return result; +} + + +void LocalStore::ensurePath(const Path & path) +{ + /* If the path is already valid, we're done. */ + if (isValidPath(path)) return; + + primeCache(*this, {path}); + + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) + throw Error(worker.exitStatus(), "path '%s' does not exist and cannot be created", path); +} + + +void LocalStore::repairPath(const Path & path) +{ + Worker worker(*this); + GoalPtr goal = worker.makeSubstitutionGoal(path, Repair); + Goals goals = {goal}; + + worker.run(goals); + + if (goal->getExitCode() != Goal::ecSuccess) { + /* Since substituting the path didn't work, if we have a valid + deriver, then rebuild the deriver. */ + auto deriver = queryPathInfo(path)->deriver; + if (deriver != "" && isValidPath(deriver)) { + goals.clear(); + goals.insert(worker.makeDerivationGoal(deriver, StringSet(), bmRepair)); + worker.run(goals); + } else + throw Error(worker.exitStatus(), "cannot repair path '%s'", path); + } +} + + +} diff --git a/third_party/nix/src/libstore/builtins.hh b/third_party/nix/src/libstore/builtins.hh new file mode 100644 index 000000000..0d2da873e --- /dev/null +++ b/third_party/nix/src/libstore/builtins.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "derivations.hh" + +namespace nix { + +// TODO: make pluggable. +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData); +void builtinBuildenv(const BasicDerivation & drv); + +} diff --git a/third_party/nix/src/libstore/builtins/buildenv.cc b/third_party/nix/src/libstore/builtins/buildenv.cc new file mode 100644 index 000000000..74e706664 --- /dev/null +++ b/third_party/nix/src/libstore/builtins/buildenv.cc @@ -0,0 +1,204 @@ +#include "builtins.hh" + +#include +#include +#include +#include + +namespace nix { + +typedef std::map Priorities; + +// FIXME: change into local variables. + +static Priorities priorities; + +static unsigned long symlinks; + +/* For each activated package, create symlinks */ +static void createLinks(const Path & srcDir, const Path & dstDir, int priority) +{ + DirEntries srcFiles; + + try { + srcFiles = readDirectory(srcDir); + } catch (SysError & e) { + if (e.errNo == ENOTDIR) { + printError("warning: not including '%s' in the user environment because it's not a directory", srcDir); + return; + } + throw; + } + + for (const auto & ent : srcFiles) { + if (ent.name[0] == '.') + /* not matched by glob */ + continue; + auto srcFile = srcDir + "/" + ent.name; + auto dstFile = dstDir + "/" + ent.name; + + struct stat srcSt; + try { + if (stat(srcFile.c_str(), &srcSt) == -1) + throw SysError("getting status of '%1%'", srcFile); + } catch (SysError & e) { + if (e.errNo == ENOENT || e.errNo == ENOTDIR) { + printError("warning: skipping dangling symlink '%s'", dstFile); + continue; + } + throw; + } + + /* The files below are special-cased to that they don't show up + * in user profiles, either because they are useless, or + * because they would cauase pointless collisions (e.g., each + * Python package brings its own + * `$out/lib/pythonX.Y/site-packages/easy-install.pth'.) + */ + if (hasSuffix(srcFile, "/propagated-build-inputs") || + hasSuffix(srcFile, "/nix-support") || + hasSuffix(srcFile, "/perllocal.pod") || + hasSuffix(srcFile, "/info/dir") || + hasSuffix(srcFile, "/log")) + continue; + + else if (S_ISDIR(srcSt.st_mode)) { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISDIR(dstSt.st_mode)) { + createLinks(srcFile, dstFile, priority); + continue; + } else if (S_ISLNK(dstSt.st_mode)) { + auto target = canonPath(dstFile, true); + if (!S_ISDIR(lstat(target).st_mode)) + throw Error("collision between '%1%' and non-directory '%2%'", srcFile, target); + if (unlink(dstFile.c_str()) == -1) + throw SysError(format("unlinking '%1%'") % dstFile); + if (mkdir(dstFile.c_str(), 0755) == -1) + throw SysError(format("creating directory '%1%'")); + createLinks(target, dstFile, priorities[dstFile]); + createLinks(srcFile, dstFile, priority); + continue; + } + } else if (errno != ENOENT) + throw SysError(format("getting status of '%1%'") % dstFile); + } + + else { + struct stat dstSt; + auto res = lstat(dstFile.c_str(), &dstSt); + if (res == 0) { + if (S_ISLNK(dstSt.st_mode)) { + auto prevPriority = priorities[dstFile]; + if (prevPriority == priority) + throw Error( + "packages '%1%' and '%2%' have the same priority %3%; " + "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' " + "to change the priority of one of the conflicting packages" + " (0 being the highest priority)", + srcFile, readLink(dstFile), priority); + if (prevPriority < priority) + continue; + if (unlink(dstFile.c_str()) == -1) + throw SysError(format("unlinking '%1%'") % dstFile); + } else if (S_ISDIR(dstSt.st_mode)) + throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); + } else if (errno != ENOENT) + throw SysError(format("getting status of '%1%'") % dstFile); + } + + createSymlink(srcFile, dstFile); + priorities[dstFile] = priority; + symlinks++; + } +} + +typedef std::set FileProp; + +static FileProp done; +static FileProp postponed = FileProp{}; + +static Path out; + +static void addPkg(const Path & pkgDir, int priority) +{ + if (done.count(pkgDir)) return; + done.insert(pkgDir); + createLinks(pkgDir, out, priority); + + try { + for (const auto & p : tokenizeString>( + readFile(pkgDir + "/nix-support/propagated-user-env-packages"), " \n")) + if (!done.count(p)) + postponed.insert(p); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != ENOTDIR) throw; + } +} + +struct Package { + Path path; + bool active; + int priority; + Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} +}; + +typedef std::vector Packages; + +void builtinBuildenv(const BasicDerivation & drv) +{ + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + out = getAttr("out"); + createDirs(out); + + /* Convert the stuff we get from the environment back into a + * coherent data type. */ + Packages pkgs; + auto derivations = tokenizeString(getAttr("derivations")); + while (!derivations.empty()) { + /* !!! We're trusting the caller to structure derivations env var correctly */ + auto active = derivations.front(); derivations.pop_front(); + auto priority = stoi(derivations.front()); derivations.pop_front(); + auto outputs = stoi(derivations.front()); derivations.pop_front(); + for (auto n = 0; n < outputs; n++) { + auto path = derivations.front(); derivations.pop_front(); + pkgs.emplace_back(path, active != "false", priority); + } + } + + /* Symlink to the packages that have been installed explicitly by the + * user. Process in priority order to reduce unnecessary + * symlink/unlink steps. + */ + std::sort(pkgs.begin(), pkgs.end(), [](const Package & a, const Package & b) { + return a.priority < b.priority || (a.priority == b.priority && a.path < b.path); + }); + for (const auto & pkg : pkgs) + if (pkg.active) + addPkg(pkg.path, pkg.priority); + + /* Symlink to the packages that have been "propagated" by packages + * installed by the user (i.e., package X declares that it wants Y + * installed as well). We do these later because they have a lower + * priority in case of collisions. + */ + auto priorityCounter = 1000; + while (!postponed.empty()) { + auto pkgDirs = postponed; + postponed = FileProp{}; + for (const auto & pkgDir : pkgDirs) + addPkg(pkgDir, priorityCounter++); + } + + printError("created %d symlinks in user environment", symlinks); + + createSymlink(getAttr("manifest"), out + "/manifest.nix"); +} + +} diff --git a/third_party/nix/src/libstore/builtins/fetchurl.cc b/third_party/nix/src/libstore/builtins/fetchurl.cc new file mode 100644 index 000000000..b1af3b4fc --- /dev/null +++ b/third_party/nix/src/libstore/builtins/fetchurl.cc @@ -0,0 +1,78 @@ +#include "builtins.hh" +#include "download.hh" +#include "store-api.hh" +#include "archive.hh" +#include "compression.hh" + +namespace nix { + +void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) +{ + /* Make the host's netrc data available. Too bad curl requires + this to be stored in a file. It would be nice if we could just + pass a pointer to the data. */ + if (netrcData != "") { + settings.netrcFile = "netrc"; + writeFile(settings.netrcFile, netrcData, 0600); + } + + auto getAttr = [&](const string & name) { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error(format("attribute '%s' missing") % name); + return i->second; + }; + + Path storePath = getAttr("out"); + auto mainUrl = getAttr("url"); + bool unpack = get(drv.env, "unpack", "") == "1"; + + /* Note: have to use a fresh downloader here because we're in + a forked process. */ + auto downloader = makeDownloader(); + + auto fetch = [&](const std::string & url) { + + auto source = sinkToSource([&](Sink & sink) { + + /* No need to do TLS verification, because we check the hash of + the result anyway. */ + DownloadRequest request(url); + request.verifyTLS = false; + request.decompress = false; + + auto decompressor = makeDecompressionSink( + unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); + downloader->download(std::move(request), *decompressor); + decompressor->finish(); + }); + + if (unpack) + restorePath(storePath, *source); + else + writeFile(storePath, *source); + + auto executable = drv.env.find("executable"); + if (executable != drv.env.end() && executable->second == "1") { + if (chmod(storePath.c_str(), 0755) == -1) + throw SysError(format("making '%1%' executable") % storePath); + } + }; + + /* Try the hashed mirrors first. */ + if (getAttr("outputHashMode") == "flat") + for (auto hashedMirror : settings.hashedMirrors.get()) + try { + if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; + auto ht = parseHashType(getAttr("outputHashAlgo")); + auto h = Hash(getAttr("outputHash"), ht); + fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(Base16, false)); + return; + } catch (Error & e) { + debug(e.what()); + } + + /* Otherwise try the specified URL. */ + fetch(mainUrl); +} + +} diff --git a/third_party/nix/src/libstore/crypto.cc b/third_party/nix/src/libstore/crypto.cc new file mode 100644 index 000000000..9ec8abd22 --- /dev/null +++ b/third_party/nix/src/libstore/crypto.cc @@ -0,0 +1,126 @@ +#include "crypto.hh" +#include "util.hh" +#include "globals.hh" + +#if HAVE_SODIUM +#include +#endif + +namespace nix { + +static std::pair split(const string & s) +{ + size_t colon = s.find(':'); + if (colon == std::string::npos || colon == 0) + return {"", ""}; + return {std::string(s, 0, colon), std::string(s, colon + 1)}; +} + +Key::Key(const string & s) +{ + auto ss = split(s); + + name = ss.first; + key = ss.second; + + if (name == "" || key == "") + throw Error("secret key is corrupt"); + + key = base64Decode(key); +} + +SecretKey::SecretKey(const string & s) + : Key(s) +{ +#if HAVE_SODIUM + if (key.size() != crypto_sign_SECRETKEYBYTES) + throw Error("secret key is not valid"); +#endif +} + +#if !HAVE_SODIUM +[[noreturn]] static void noSodium() +{ + throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +} +#endif + +std::string SecretKey::signDetached(const std::string & data) const +{ +#if HAVE_SODIUM + unsigned char sig[crypto_sign_BYTES]; + unsigned long long sigLen; + crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), + (unsigned char *) key.data()); + return name + ":" + base64Encode(std::string((char *) sig, sigLen)); +#else + noSodium(); +#endif +} + +PublicKey SecretKey::toPublicKey() const +{ +#if HAVE_SODIUM + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); + return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); +#else + noSodium(); +#endif +} + +PublicKey::PublicKey(const string & s) + : Key(s) +{ +#if HAVE_SODIUM + if (key.size() != crypto_sign_PUBLICKEYBYTES) + throw Error("public key is not valid"); +#endif +} + +bool verifyDetached(const std::string & data, const std::string & sig, + const PublicKeys & publicKeys) +{ +#if HAVE_SODIUM + auto ss = split(sig); + + auto key = publicKeys.find(ss.first); + if (key == publicKeys.end()) return false; + + auto sig2 = base64Decode(ss.second); + if (sig2.size() != crypto_sign_BYTES) + throw Error("signature is not valid"); + + return crypto_sign_verify_detached((unsigned char *) sig2.data(), + (unsigned char *) data.data(), data.size(), + (unsigned char *) key->second.key.data()) == 0; +#else + noSodium(); +#endif +} + +PublicKeys getDefaultPublicKeys() +{ + PublicKeys publicKeys; + + // FIXME: filter duplicates + + for (auto s : settings.trustedPublicKeys.get()) { + PublicKey key(s); + publicKeys.emplace(key.name, key); + } + + for (auto secretKeyFile : settings.secretKeyFiles.get()) { + try { + SecretKey secretKey(readFile(secretKeyFile)); + publicKeys.emplace(secretKey.name, secretKey.toPublicKey()); + } catch (SysError & e) { + /* Ignore unreadable key files. That's normal in a + multi-user installation. */ + } + } + + return publicKeys; +} + +} diff --git a/third_party/nix/src/libstore/crypto.hh b/third_party/nix/src/libstore/crypto.hh new file mode 100644 index 000000000..9110af3aa --- /dev/null +++ b/third_party/nix/src/libstore/crypto.hh @@ -0,0 +1,54 @@ +#pragma once + +#include "types.hh" + +#include + +namespace nix { + +struct Key +{ + std::string name; + std::string key; + + /* Construct Key from a string in the format + ‘:’. */ + Key(const std::string & s); + +protected: + Key(const std::string & name, const std::string & key) + : name(name), key(key) { } +}; + +struct PublicKey; + +struct SecretKey : Key +{ + SecretKey(const std::string & s); + + /* Return a detached signature of the given string. */ + std::string signDetached(const std::string & s) const; + + PublicKey toPublicKey() const; +}; + +struct PublicKey : Key +{ + PublicKey(const std::string & data); + +private: + PublicKey(const std::string & name, const std::string & key) + : Key(name, key) { } + friend struct SecretKey; +}; + +typedef std::map PublicKeys; + +/* Return true iff ‘sig’ is a correct signature over ‘data’ using one + of the given public keys. */ +bool verifyDetached(const std::string & data, const std::string & sig, + const PublicKeys & publicKeys); + +PublicKeys getDefaultPublicKeys(); + +} diff --git a/third_party/nix/src/libstore/derivations.cc b/third_party/nix/src/libstore/derivations.cc new file mode 100644 index 000000000..23fcfb281 --- /dev/null +++ b/third_party/nix/src/libstore/derivations.cc @@ -0,0 +1,416 @@ +#include "derivations.hh" +#include "store-api.hh" +#include "globals.hh" +#include "util.hh" +#include "worker-protocol.hh" +#include "fs-accessor.hh" +#include "istringstream_nocopy.hh" + +namespace nix { + + +void DerivationOutput::parseHashInfo(bool & recursive, Hash & hash) const +{ + recursive = false; + string algo = hashAlgo; + + if (string(algo, 0, 2) == "r:") { + recursive = true; + algo = string(algo, 2); + } + + HashType hashType = parseHashType(algo); + if (hashType == htUnknown) + throw Error(format("unknown hash algorithm '%1%'") % algo); + + hash = Hash(this->hash, hashType); +} + + +Path BasicDerivation::findOutput(const string & id) const +{ + auto i = outputs.find(id); + if (i == outputs.end()) + throw Error(format("derivation has no output '%1%'") % id); + return i->second.path; +} + + +bool BasicDerivation::isBuiltin() const +{ + return string(builder, 0, 8) == "builtin:"; +} + + +Path writeDerivation(ref store, + const Derivation & drv, const string & name, RepairFlag repair) +{ + PathSet references; + references.insert(drv.inputSrcs.begin(), drv.inputSrcs.end()); + for (auto & i : drv.inputDrvs) + references.insert(i.first); + /* Note that the outputs of a derivation are *not* references + (that can be missing (of course) and should not necessarily be + held during a garbage collection). */ + string suffix = name + drvExtension; + string contents = drv.unparse(); + return settings.readOnlyMode + ? store->computeStorePathForText(suffix, contents, references) + : store->addTextToStore(suffix, contents, references, repair); +} + + +/* Read string `s' from stream `str'. */ +static void expect(std::istream & str, const string & s) +{ + char s2[s.size()]; + str.read(s2, s.size()); + if (string(s2, s.size()) != s) + throw FormatError(format("expected string '%1%'") % s); +} + + +/* Read a C-style string from stream `str'. */ +static string parseString(std::istream & str) +{ + string res; + expect(str, "\""); + int c; + while ((c = str.get()) != '"') + if (c == '\\') { + c = str.get(); + if (c == 'n') res += '\n'; + else if (c == 'r') res += '\r'; + else if (c == 't') res += '\t'; + else res += c; + } + else res += c; + return res; +} + + +static Path parsePath(std::istream & str) +{ + string s = parseString(str); + if (s.size() == 0 || s[0] != '/') + throw FormatError(format("bad path '%1%' in derivation") % s); + return s; +} + + +static bool endOfList(std::istream & str) +{ + if (str.peek() == ',') { + str.get(); + return false; + } + if (str.peek() == ']') { + str.get(); + return true; + } + return false; +} + + +static StringSet parseStrings(std::istream & str, bool arePaths) +{ + StringSet res; + while (!endOfList(str)) + res.insert(arePaths ? parsePath(str) : parseString(str)); + return res; +} + + +static Derivation parseDerivation(const string & s) +{ + Derivation drv; + istringstream_nocopy str(s); + expect(str, "Derive(["); + + /* Parse the list of outputs. */ + while (!endOfList(str)) { + DerivationOutput out; + expect(str, "("); string id = parseString(str); + expect(str, ","); out.path = parsePath(str); + expect(str, ","); out.hashAlgo = parseString(str); + expect(str, ","); out.hash = parseString(str); + expect(str, ")"); + drv.outputs[id] = out; + } + + /* Parse the list of input derivations. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); + Path drvPath = parsePath(str); + expect(str, ",["); + drv.inputDrvs[drvPath] = parseStrings(str, false); + expect(str, ")"); + } + + expect(str, ",["); drv.inputSrcs = parseStrings(str, true); + expect(str, ","); drv.platform = parseString(str); + expect(str, ","); drv.builder = parseString(str); + + /* Parse the builder arguments. */ + expect(str, ",["); + while (!endOfList(str)) + drv.args.push_back(parseString(str)); + + /* Parse the environment variables. */ + expect(str, ",["); + while (!endOfList(str)) { + expect(str, "("); string name = parseString(str); + expect(str, ","); string value = parseString(str); + expect(str, ")"); + drv.env[name] = value; + } + + expect(str, ")"); + return drv; +} + + +Derivation readDerivation(const Path & drvPath) +{ + try { + return parseDerivation(readFile(drvPath)); + } catch (FormatError & e) { + throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + } +} + + +Derivation Store::derivationFromPath(const Path & drvPath) +{ + assertStorePath(drvPath); + ensurePath(drvPath); + auto accessor = getFSAccessor(); + try { + return parseDerivation(accessor->readFile(drvPath)); + } catch (FormatError & e) { + throw Error(format("error parsing derivation '%1%': %2%") % drvPath % e.msg()); + } +} + + +static void printString(string & res, const string & s) +{ + res += '"'; + for (const char * i = s.c_str(); *i; i++) + if (*i == '\"' || *i == '\\') { res += "\\"; res += *i; } + else if (*i == '\n') res += "\\n"; + else if (*i == '\r') res += "\\r"; + else if (*i == '\t') res += "\\t"; + else res += *i; + res += '"'; +} + + +template +static void printStrings(string & res, ForwardIterator i, ForwardIterator j) +{ + res += '['; + bool first = true; + for ( ; i != j; ++i) { + if (first) first = false; else res += ','; + printString(res, *i); + } + res += ']'; +} + + +string Derivation::unparse() const +{ + string s; + s.reserve(65536); + s += "Derive(["; + + bool first = true; + for (auto & i : outputs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printString(s, i.second.path); + s += ','; printString(s, i.second.hashAlgo); + s += ','; printString(s, i.second.hash); + s += ')'; + } + + s += "],["; + first = true; + for (auto & i : inputDrvs) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printStrings(s, i.second.begin(), i.second.end()); + s += ')'; + } + + s += "],"; + printStrings(s, inputSrcs.begin(), inputSrcs.end()); + + s += ','; printString(s, platform); + s += ','; printString(s, builder); + s += ','; printStrings(s, args.begin(), args.end()); + + s += ",["; + first = true; + for (auto & i : env) { + if (first) first = false; else s += ','; + s += '('; printString(s, i.first); + s += ','; printString(s, i.second); + s += ')'; + } + + s += "])"; + + return s; +} + + +bool isDerivation(const string & fileName) +{ + return hasSuffix(fileName, drvExtension); +} + + +bool BasicDerivation::isFixedOutput() const +{ + return outputs.size() == 1 && + outputs.begin()->first == "out" && + outputs.begin()->second.hash != ""; +} + + +DrvHashes drvHashes; + + +/* Returns the hash of a derivation modulo fixed-output + subderivations. A fixed-output derivation is a derivation with one + output (`out') for which an expected hash and hash algorithm are + specified (using the `outputHash' and `outputHashAlgo' + attributes). We don't want changes to such derivations to + propagate upwards through the dependency graph, changing output + paths everywhere. + + For instance, if we change the url in a call to the `fetchurl' + function, we do not want to rebuild everything depending on it + (after all, (the hash of) the file being downloaded is unchanged). + So the *output paths* should not change. On the other hand, the + *derivation paths* should change to reflect the new dependency + graph. + + That's what this function does: it returns a hash which is just the + hash of the derivation ATerm, except that any input derivation + paths have been replaced by the result of a recursive call to this + function, and that for fixed-output derivations we return a hash of + its output path. */ +Hash hashDerivationModulo(Store & store, Derivation drv) +{ + /* Return a fixed hash for fixed-output derivations. */ + if (drv.isFixedOutput()) { + DerivationOutputs::const_iterator i = drv.outputs.begin(); + return hashString(htSHA256, "fixed:out:" + + i->second.hashAlgo + ":" + + i->second.hash + ":" + + i->second.path); + } + + /* For other derivations, replace the inputs paths with recursive + calls to this function.*/ + DerivationInputs inputs2; + for (auto & i : drv.inputDrvs) { + Hash h = drvHashes[i.first]; + if (!h) { + assert(store.isValidPath(i.first)); + Derivation drv2 = readDerivation(store.toRealPath(i.first)); + h = hashDerivationModulo(store, drv2); + drvHashes[i.first] = h; + } + inputs2[h.to_string(Base16, false)] = i.second; + } + drv.inputDrvs = inputs2; + + return hashString(htSHA256, drv.unparse()); +} + + +DrvPathWithOutputs parseDrvPathWithOutputs(const string & s) +{ + size_t n = s.find("!"); + return n == s.npos + ? DrvPathWithOutputs(s, std::set()) + : DrvPathWithOutputs(string(s, 0, n), tokenizeString >(string(s, n + 1), ",")); +} + + +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set & outputs) +{ + return outputs.empty() + ? drvPath + : drvPath + "!" + concatStringsSep(",", outputs); +} + + +bool wantOutput(const string & output, const std::set & wanted) +{ + return wanted.empty() || wanted.find(output) != wanted.end(); +} + + +PathSet BasicDerivation::outputPaths() const +{ + PathSet paths; + for (auto & i : outputs) + paths.insert(i.second.path); + return paths; +} + + +Source & readDerivation(Source & in, Store & store, BasicDerivation & drv) +{ + drv.outputs.clear(); + auto nr = readNum(in); + for (size_t n = 0; n < nr; n++) { + auto name = readString(in); + DerivationOutput o; + in >> o.path >> o.hashAlgo >> o.hash; + store.assertStorePath(o.path); + drv.outputs[name] = o; + } + + drv.inputSrcs = readStorePaths(store, in); + in >> drv.platform >> drv.builder; + drv.args = readStrings(in); + + nr = readNum(in); + for (size_t n = 0; n < nr; n++) { + auto key = readString(in); + auto value = readString(in); + drv.env[key] = value; + } + + return in; +} + + +Sink & operator << (Sink & out, const BasicDerivation & drv) +{ + out << drv.outputs.size(); + for (auto & i : drv.outputs) + out << i.first << i.second.path << i.second.hashAlgo << i.second.hash; + out << drv.inputSrcs << drv.platform << drv.builder << drv.args; + out << drv.env.size(); + for (auto & i : drv.env) + out << i.first << i.second; + return out; +} + + +std::string hashPlaceholder(const std::string & outputName) +{ + // FIXME: memoize? + return "/" + hashString(htSHA256, "nix-output:" + outputName).to_string(Base32, false); +} + + +} diff --git a/third_party/nix/src/libstore/derivations.hh b/third_party/nix/src/libstore/derivations.hh new file mode 100644 index 000000000..8e02c9bc5 --- /dev/null +++ b/third_party/nix/src/libstore/derivations.hh @@ -0,0 +1,117 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "store-api.hh" + +#include + + +namespace nix { + + +/* Extension of derivations in the Nix store. */ +const string drvExtension = ".drv"; + + +/* Abstract syntax of derivations. */ + +struct DerivationOutput +{ + Path path; + string hashAlgo; /* hash used for expected hash computation */ + string hash; /* expected hash, may be null */ + DerivationOutput() + { + } + DerivationOutput(Path path, string hashAlgo, string hash) + { + this->path = path; + this->hashAlgo = hashAlgo; + this->hash = hash; + } + void parseHashInfo(bool & recursive, Hash & hash) const; +}; + +typedef std::map DerivationOutputs; + +/* For inputs that are sub-derivations, we specify exactly which + output IDs we are interested in. */ +typedef std::map DerivationInputs; + +typedef std::map StringPairs; + +struct BasicDerivation +{ + DerivationOutputs outputs; /* keyed on symbolic IDs */ + PathSet inputSrcs; /* inputs that are sources */ + string platform; + Path builder; + Strings args; + StringPairs env; + + virtual ~BasicDerivation() { }; + + /* Return the path corresponding to the output identifier `id' in + the given derivation. */ + Path findOutput(const string & id) const; + + bool isBuiltin() const; + + /* Return true iff this is a fixed-output derivation. */ + bool isFixedOutput() const; + + /* Return the output paths of a derivation. */ + PathSet outputPaths() const; + +}; + +struct Derivation : BasicDerivation +{ + DerivationInputs inputDrvs; /* inputs that are sub-derivations */ + + /* Print a derivation. */ + std::string unparse() const; +}; + + +class Store; + + +/* Write a derivation to the Nix store, and return its path. */ +Path writeDerivation(ref store, + const Derivation & drv, const string & name, RepairFlag repair = NoRepair); + +/* Read a derivation from a file. */ +Derivation readDerivation(const Path & drvPath); + +/* Check whether a file name ends with the extension for + derivations. */ +bool isDerivation(const string & fileName); + +Hash hashDerivationModulo(Store & store, Derivation drv); + +/* Memoisation of hashDerivationModulo(). */ +typedef std::map DrvHashes; + +extern DrvHashes drvHashes; // FIXME: global, not thread-safe + +/* Split a string specifying a derivation and a set of outputs + (/nix/store/hash-foo!out1,out2,...) into the derivation path and + the outputs. */ +typedef std::pair > DrvPathWithOutputs; +DrvPathWithOutputs parseDrvPathWithOutputs(const string & s); + +Path makeDrvPathWithOutputs(const Path & drvPath, const std::set & outputs); + +bool wantOutput(const string & output, const std::set & wanted); + +struct Source; +struct Sink; + +Source & readDerivation(Source & in, Store & store, BasicDerivation & drv); +Sink & operator << (Sink & out, const BasicDerivation & drv); + +std::string hashPlaceholder(const std::string & outputName); + +} diff --git a/third_party/nix/src/libstore/download.cc b/third_party/nix/src/libstore/download.cc new file mode 100644 index 000000000..80674a9e7 --- /dev/null +++ b/third_party/nix/src/libstore/download.cc @@ -0,0 +1,946 @@ +#include "download.hh" +#include "util.hh" +#include "globals.hh" +#include "hash.hh" +#include "store-api.hh" +#include "archive.hh" +#include "s3.hh" +#include "compression.hh" +#include "pathlocks.hh" +#include "finally.hh" + +#ifdef ENABLE_S3 +#include +#endif + +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +using namespace std::string_literals; + +namespace nix { + +DownloadSettings downloadSettings; + +static GlobalConfig::Register r1(&downloadSettings); + +std::string resolveUri(const std::string & uri) +{ + if (uri.compare(0, 8, "channel:") == 0) + return "https://nixos.org/channels/" + std::string(uri, 8) + "/nixexprs.tar.xz"; + else + return uri; +} + +struct CurlDownloader : public Downloader +{ + CURLM * curlm = 0; + + std::random_device rd; + std::mt19937 mt19937; + + struct DownloadItem : public std::enable_shared_from_this + { + CurlDownloader & downloader; + DownloadRequest request; + DownloadResult result; + Activity act; + bool done = false; // whether either the success or failure function has been called + Callback callback; + CURL * req = 0; + bool active = false; // whether the handle has been added to the multi object + std::string status; + + unsigned int attempt = 0; + + /* Don't start this download until the specified time point + has been reached. */ + std::chrono::steady_clock::time_point embargo; + + struct curl_slist * requestHeaders = 0; + + std::string encoding; + + bool acceptRanges = false; + + curl_off_t writtenToSink = 0; + + DownloadItem(CurlDownloader & downloader, + const DownloadRequest & request, + Callback && callback) + : downloader(downloader) + , request(request) + , act(*logger, lvlTalkative, actDownload, + fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri), + {request.uri}, request.parentAct) + , callback(std::move(callback)) + , finalSink([this](const unsigned char * data, size_t len) { + if (this->request.dataCallback) { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + /* Only write data to the sink if this is a + successful response. */ + if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) { + writtenToSink += len; + this->request.dataCallback((char *) data, len); + } + } else + this->result.data->append((char *) data, len); + }) + { + if (!request.expectedETag.empty()) + requestHeaders = curl_slist_append(requestHeaders, ("If-None-Match: " + request.expectedETag).c_str()); + if (!request.mimeType.empty()) + requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str()); + } + + ~DownloadItem() + { + if (req) { + if (active) + curl_multi_remove_handle(downloader.curlm, req); + curl_easy_cleanup(req); + } + if (requestHeaders) curl_slist_free_all(requestHeaders); + try { + if (!done) + fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri)); + } catch (...) { + ignoreException(); + } + } + + void failEx(std::exception_ptr ex) + { + assert(!done); + done = true; + callback.rethrow(ex); + } + + template + void fail(const T & e) + { + failEx(std::make_exception_ptr(e)); + } + + LambdaSink finalSink; + std::shared_ptr decompressionSink; + + std::exception_ptr writeException; + + size_t writeCallback(void * contents, size_t size, size_t nmemb) + { + try { + size_t realSize = size * nmemb; + result.bodySize += realSize; + + if (!decompressionSink) + decompressionSink = makeDecompressionSink(encoding, finalSink); + + (*decompressionSink)((unsigned char *) contents, realSize); + + return realSize; + } catch (...) { + writeException = std::current_exception(); + return 0; + } + } + + static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb); + } + + size_t headerCallback(void * contents, size_t size, size_t nmemb) + { + size_t realSize = size * nmemb; + std::string line((char *) contents, realSize); + printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line)); + if (line.compare(0, 5, "HTTP/") == 0) { // new response starts + result.etag = ""; + auto ss = tokenizeString>(line, " "); + status = ss.size() >= 2 ? ss[1] : ""; + result.data = std::make_shared(); + result.bodySize = 0; + acceptRanges = false; + encoding = ""; + } else { + auto i = line.find(':'); + if (i != string::npos) { + string name = toLower(trim(string(line, 0, i))); + if (name == "etag") { + result.etag = trim(string(line, i + 1)); + /* Hack to work around a GitHub bug: it sends + ETags, but ignores If-None-Match. So if we get + the expected ETag on a 200 response, then shut + down the connection because we already have the + data. */ + if (result.etag == request.expectedETag && status == "200") { + debug(format("shutting down on 200 HTTP response with expected ETag")); + return 0; + } + } else if (name == "content-encoding") + encoding = trim(string(line, i + 1)); + else if (name == "accept-ranges" && toLower(trim(std::string(line, i + 1))) == "bytes") + acceptRanges = true; + } + } + return realSize; + } + + static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp) + { + return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb); + } + + int progressCallback(double dltotal, double dlnow) + { + try { + act.progress(dlnow, dltotal); + } catch (nix::Interrupted &) { + assert(_isInterrupted); + } + return _isInterrupted; + } + + static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) + { + return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow); + } + + static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr) + { + if (type == CURLINFO_TEXT) + vomit("curl: %s", chomp(std::string(data, size))); + return 0; + } + + size_t readOffset = 0; + size_t readCallback(char *buffer, size_t size, size_t nitems) + { + if (readOffset == request.data->length()) + return 0; + auto count = std::min(size * nitems, request.data->length() - readOffset); + assert(count); + memcpy(buffer, request.data->data() + readOffset, count); + readOffset += count; + return count; + } + + static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp) + { + return ((DownloadItem *) userp)->readCallback(buffer, size, nitems); + } + + void init() + { + if (!req) req = curl_easy_init(); + + curl_easy_reset(req); + + if (verbosity >= lvlVomit) { + curl_easy_setopt(req, CURLOPT_VERBOSE, 1); + curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback); + } + + curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); + curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); + curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); + curl_easy_setopt(req, CURLOPT_USERAGENT, + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + + (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str()); + #if LIBCURL_VERSION_NUM >= 0x072b00 + curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1); + #endif + #if LIBCURL_VERSION_NUM >= 0x072f00 + if (downloadSettings.enableHttp2) + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS); + else + curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + #endif + curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper); + curl_easy_setopt(req, CURLOPT_WRITEDATA, this); + curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper); + curl_easy_setopt(req, CURLOPT_HEADERDATA, this); + + curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper); + curl_easy_setopt(req, CURLOPT_PROGRESSDATA, this); + curl_easy_setopt(req, CURLOPT_NOPROGRESS, 0); + + curl_easy_setopt(req, CURLOPT_HTTPHEADER, requestHeaders); + + if (request.head) + curl_easy_setopt(req, CURLOPT_NOBODY, 1); + + if (request.data) { + curl_easy_setopt(req, CURLOPT_UPLOAD, 1L); + curl_easy_setopt(req, CURLOPT_READFUNCTION, readCallbackWrapper); + curl_easy_setopt(req, CURLOPT_READDATA, this); + curl_easy_setopt(req, CURLOPT_INFILESIZE_LARGE, (curl_off_t) request.data->length()); + } + + if (request.verifyTLS) { + if (settings.caFile != "") + curl_easy_setopt(req, CURLOPT_CAINFO, settings.caFile.c_str()); + } else { + curl_easy_setopt(req, CURLOPT_SSL_VERIFYPEER, 0); + curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0); + } + + curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get()); + + curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L); + curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get()); + + /* If no file exist in the specified path, curl continues to work + anyway as if netrc support was disabled. */ + curl_easy_setopt(req, CURLOPT_NETRC_FILE, settings.netrcFile.get().c_str()); + curl_easy_setopt(req, CURLOPT_NETRC, CURL_NETRC_OPTIONAL); + + if (writtenToSink) + curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + + result.data = std::make_shared(); + result.bodySize = 0; + } + + void finish(CURLcode code) + { + long httpStatus = 0; + curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus); + + char * effectiveUriCStr; + curl_easy_getinfo(req, CURLINFO_EFFECTIVE_URL, &effectiveUriCStr); + if (effectiveUriCStr) + result.effectiveUri = effectiveUriCStr; + + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", + request.verb(), request.uri, code, httpStatus, result.bodySize); + + if (decompressionSink) { + try { + decompressionSink->finish(); + } catch (...) { + writeException = std::current_exception(); + } + } + + if (code == CURLE_WRITE_ERROR && result.etag == request.expectedETag) { + code = CURLE_OK; + httpStatus = 304; + } + + if (writeException) + failEx(writeException); + + else if (code == CURLE_OK && + (httpStatus == 200 || httpStatus == 201 || httpStatus == 204 || httpStatus == 206 || httpStatus == 304 || httpStatus == 226 /* FTP */ || httpStatus == 0 /* other protocol */)) + { + result.cached = httpStatus == 304; + act.progress(result.bodySize, result.bodySize); + done = true; + callback(std::move(result)); + } + + else { + // We treat most errors as transient, but won't retry when hopeless + Error err = Transient; + + if (httpStatus == 404 || httpStatus == 410 || code == CURLE_FILE_COULDNT_READ_FILE) { + // The file is definitely not there + err = NotFound; + } else if (httpStatus == 401 || httpStatus == 403 || httpStatus == 407) { + // Don't retry on authentication/authorization failures + err = Forbidden; + } else if (httpStatus >= 400 && httpStatus < 500 && httpStatus != 408 && httpStatus != 429) { + // Most 4xx errors are client errors and are probably not worth retrying: + // * 408 means the server timed out waiting for us, so we try again + // * 429 means too many requests, so we retry (with a delay) + err = Misc; + } else if (httpStatus == 501 || httpStatus == 505 || httpStatus == 511) { + // Let's treat most 5xx (server) errors as transient, except for a handful: + // * 501 not implemented + // * 505 http version not supported + // * 511 we're behind a captive portal + err = Misc; + } else { + // Don't bother retrying on certain cURL errors either + switch (code) { + case CURLE_FAILED_INIT: + case CURLE_URL_MALFORMAT: + case CURLE_NOT_BUILT_IN: + case CURLE_REMOTE_ACCESS_DENIED: + case CURLE_FILE_COULDNT_READ_FILE: + case CURLE_FUNCTION_NOT_FOUND: + case CURLE_ABORTED_BY_CALLBACK: + case CURLE_BAD_FUNCTION_ARGUMENT: + case CURLE_INTERFACE_FAILED: + case CURLE_UNKNOWN_OPTION: + case CURLE_SSL_CACERT_BADFILE: + case CURLE_TOO_MANY_REDIRECTS: + case CURLE_WRITE_ERROR: + case CURLE_UNSUPPORTED_PROTOCOL: + err = Misc; + break; + default: // Shut up warnings + break; + } + } + + attempt++; + + auto exc = + code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted + ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri)) + : httpStatus != 0 + ? DownloadError(err, + fmt("unable to %s '%s': HTTP error %d", + request.verb(), request.uri, httpStatus) + + (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) + ) + : DownloadError(err, + fmt("unable to %s '%s': %s (%d)", + request.verb(), request.uri, curl_easy_strerror(code), code)); + + /* If this is a transient error, then maybe retry the + download after a while. If we're writing to a + sink, we can only retry if the server supports + ranged requests. */ + if (err == Transient + && attempt < request.tries + && (!this->request.dataCallback + || writtenToSink == 0 + || (acceptRanges && encoding.empty()))) + { + int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937)); + if (writtenToSink) + warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms); + else + warn("%s; retrying in %d ms", exc.what(), ms); + embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms); + downloader.enqueueItem(shared_from_this()); + } + else + fail(exc); + } + } + }; + + struct State + { + struct EmbargoComparator { + bool operator() (const std::shared_ptr & i1, const std::shared_ptr & i2) { + return i1->embargo > i2->embargo; + } + }; + bool quit = false; + std::priority_queue, std::vector>, EmbargoComparator> incoming; + }; + + Sync state_; + + /* We can't use a std::condition_variable to wake up the curl + thread, because it only monitors file descriptors. So use a + pipe instead. */ + Pipe wakeupPipe; + + std::thread workerThread; + + CurlDownloader() + : mt19937(rd()) + { + static std::once_flag globalInit; + std::call_once(globalInit, curl_global_init, CURL_GLOBAL_ALL); + + curlm = curl_multi_init(); + + #if LIBCURL_VERSION_NUM >= 0x072b00 // Multiplex requires >= 7.43.0 + curl_multi_setopt(curlm, CURLMOPT_PIPELINING, CURLPIPE_MULTIPLEX); + #endif + #if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0 + curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS, + downloadSettings.httpConnections.get()); + #endif + + wakeupPipe.create(); + fcntl(wakeupPipe.readSide.get(), F_SETFL, O_NONBLOCK); + + workerThread = std::thread([&]() { workerThreadEntry(); }); + } + + ~CurlDownloader() + { + stopWorkerThread(); + + workerThread.join(); + + if (curlm) curl_multi_cleanup(curlm); + } + + void stopWorkerThread() + { + /* Signal the worker thread to exit. */ + { + auto state(state_.lock()); + state->quit = true; + } + writeFull(wakeupPipe.writeSide.get(), " ", false); + } + + void workerThreadMain() + { + /* Cause this thread to be notified on SIGINT. */ + auto callback = createInterruptCallback([&]() { + stopWorkerThread(); + }); + + std::map> items; + + bool quit = false; + + std::chrono::steady_clock::time_point nextWakeup; + + while (!quit) { + checkInterrupt(); + + /* Let curl do its thing. */ + int running; + CURLMcode mc = curl_multi_perform(curlm, &running); + if (mc != CURLM_OK) + throw nix::Error(format("unexpected error from curl_multi_perform(): %s") % curl_multi_strerror(mc)); + + /* Set the promises of any finished requests. */ + CURLMsg * msg; + int left; + while ((msg = curl_multi_info_read(curlm, &left))) { + if (msg->msg == CURLMSG_DONE) { + auto i = items.find(msg->easy_handle); + assert(i != items.end()); + i->second->finish(msg->data.result); + curl_multi_remove_handle(curlm, i->second->req); + i->second->active = false; + items.erase(i); + } + } + + /* Wait for activity, including wakeup events. */ + int numfds = 0; + struct curl_waitfd extraFDs[1]; + extraFDs[0].fd = wakeupPipe.readSide.get(); + extraFDs[0].events = CURL_WAIT_POLLIN; + extraFDs[0].revents = 0; + long maxSleepTimeMs = items.empty() ? 10000 : 100; + auto sleepTimeMs = + nextWakeup != std::chrono::steady_clock::time_point() + ? std::max(0, (int) std::chrono::duration_cast(nextWakeup - std::chrono::steady_clock::now()).count()) + : maxSleepTimeMs; + vomit("download thread waiting for %d ms", sleepTimeMs); + mc = curl_multi_wait(curlm, extraFDs, 1, sleepTimeMs, &numfds); + if (mc != CURLM_OK) + throw nix::Error(format("unexpected error from curl_multi_wait(): %s") % curl_multi_strerror(mc)); + + nextWakeup = std::chrono::steady_clock::time_point(); + + /* Add new curl requests from the incoming requests queue, + except for requests that are embargoed (waiting for a + retry timeout to expire). */ + if (extraFDs[0].revents & CURL_WAIT_POLLIN) { + char buf[1024]; + auto res = read(extraFDs[0].fd, buf, sizeof(buf)); + if (res == -1 && errno != EINTR) + throw SysError("reading curl wakeup socket"); + } + + std::vector> incoming; + auto now = std::chrono::steady_clock::now(); + + { + auto state(state_.lock()); + while (!state->incoming.empty()) { + auto item = state->incoming.top(); + if (item->embargo <= now) { + incoming.push_back(item); + state->incoming.pop(); + } else { + if (nextWakeup == std::chrono::steady_clock::time_point() + || item->embargo < nextWakeup) + nextWakeup = item->embargo; + break; + } + } + quit = state->quit; + } + + for (auto & item : incoming) { + debug("starting %s of %s", item->request.verb(), item->request.uri); + item->init(); + curl_multi_add_handle(curlm, item->req); + item->active = true; + items[item->req] = item; + } + } + + debug("download thread shutting down"); + } + + void workerThreadEntry() + { + try { + workerThreadMain(); + } catch (nix::Interrupted & e) { + } catch (std::exception & e) { + printError("unexpected error in download thread: %s", e.what()); + } + + { + auto state(state_.lock()); + while (!state->incoming.empty()) state->incoming.pop(); + state->quit = true; + } + } + + void enqueueItem(std::shared_ptr item) + { + if (item->request.data + && !hasPrefix(item->request.uri, "http://") + && !hasPrefix(item->request.uri, "https://")) + throw nix::Error("uploading to '%s' is not supported", item->request.uri); + + { + auto state(state_.lock()); + if (state->quit) + throw nix::Error("cannot enqueue download request because the download thread is shutting down"); + state->incoming.push(item); + } + writeFull(wakeupPipe.writeSide.get(), " "); + } + +#ifdef ENABLE_S3 + std::tuple parseS3Uri(std::string uri) + { + auto [path, params] = splitUriAndParams(uri); + + auto slash = path.find('/', 5); // 5 is the length of "s3://" prefix + if (slash == std::string::npos) + throw nix::Error("bad S3 URI '%s'", path); + + std::string bucketName(path, 5, slash - 5); + std::string key(path, slash + 1); + + return {bucketName, key, params}; + } +#endif + + void enqueueDownload(const DownloadRequest & request, + Callback callback) override + { + /* Ugly hack to support s3:// URIs. */ + if (hasPrefix(request.uri, "s3://")) { + // FIXME: do this on a worker thread + try { +#ifdef ENABLE_S3 + auto [bucketName, key, params] = parseS3Uri(request.uri); + + std::string profile = get(params, "profile", ""); + std::string region = get(params, "region", Aws::Region::US_EAST_1); + std::string scheme = get(params, "scheme", ""); + std::string endpoint = get(params, "endpoint", ""); + + S3Helper s3Helper(profile, region, scheme, endpoint); + + // FIXME: implement ETag + auto s3Res = s3Helper.getObject(bucketName, key); + DownloadResult res; + if (!s3Res.data) + throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri)); + res.data = s3Res.data; + callback(std::move(res)); +#else + throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); +#endif + } catch (...) { callback.rethrow(); } + return; + } + + enqueueItem(std::make_shared(*this, request, std::move(callback))); + } +}; + +ref getDownloader() +{ + static ref downloader = makeDownloader(); + return downloader; +} + +ref makeDownloader() +{ + return make_ref(); +} + +std::future Downloader::enqueueDownload(const DownloadRequest & request) +{ + auto promise = std::make_shared>(); + enqueueDownload(request, + {[promise](std::future fut) { + try { + promise->set_value(fut.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + }}); + return promise->get_future(); +} + +DownloadResult Downloader::download(const DownloadRequest & request) +{ + return enqueueDownload(request).get(); +} + +void Downloader::download(DownloadRequest && request, Sink & sink) +{ + /* Note: we can't call 'sink' via request.dataCallback, because + that would cause the sink to execute on the downloader + thread. If 'sink' is a coroutine, this will fail. Also, if the + sink is expensive (e.g. one that does decompression and writing + to the Nix store), it would stall the download thread too much. + Therefore we use a buffer to communicate data between the + download thread and the calling thread. */ + + struct State { + bool quit = false; + std::exception_ptr exc; + std::string data; + std::condition_variable avail, request; + }; + + auto _state = std::make_shared>(); + + /* In case of an exception, wake up the download thread. FIXME: + abort the download request. */ + Finally finally([&]() { + auto state(_state->lock()); + state->quit = true; + state->request.notify_one(); + }); + + request.dataCallback = [_state](char * buf, size_t len) { + + auto state(_state->lock()); + + if (state->quit) return; + + /* If the buffer is full, then go to sleep until the calling + thread wakes us up (i.e. when it has removed data from the + buffer). We don't wait forever to prevent stalling the + download thread. (Hopefully sleeping will throttle the + sender.) */ + if (state->data.size() > 1024 * 1024) { + debug("download buffer is full; going to sleep"); + state.wait_for(state->request, std::chrono::seconds(10)); + } + + /* Append data to the buffer and wake up the calling + thread. */ + state->data.append(buf, len); + state->avail.notify_one(); + }; + + enqueueDownload(request, + {[_state](std::future fut) { + auto state(_state->lock()); + state->quit = true; + try { + fut.get(); + } catch (...) { + state->exc = std::current_exception(); + } + state->avail.notify_one(); + state->request.notify_one(); + }}); + + while (true) { + checkInterrupt(); + + std::string chunk; + + /* Grab data if available, otherwise wait for the download + thread to wake us up. */ + { + auto state(_state->lock()); + + while (state->data.empty()) { + + if (state->quit) { + if (state->exc) std::rethrow_exception(state->exc); + return; + } + + state.wait(state->avail); + } + + chunk = std::move(state->data); + + state->request.notify_one(); + } + + /* Flush the data to the sink and wake up the download thread + if it's blocked on a full buffer. We don't hold the state + lock while doing this to prevent blocking the download + thread if sink() takes a long time. */ + sink((unsigned char *) chunk.data(), chunk.size()); + } +} + +CachedDownloadResult Downloader::downloadCached( + ref store, const CachedDownloadRequest & request) +{ + auto url = resolveUri(request.uri); + + auto name = request.name; + if (name == "") { + auto p = url.rfind('/'); + if (p != string::npos) name = string(url, p + 1); + } + + Path expectedStorePath; + if (request.expectedHash) { + expectedStorePath = store->makeFixedOutputPath(request.unpack, request.expectedHash, name); + if (store->isValidPath(expectedStorePath)) { + CachedDownloadResult result; + result.storePath = expectedStorePath; + result.path = store->toRealPath(expectedStorePath); + return result; + } + } + + Path cacheDir = getCacheDir() + "/nix/tarballs"; + createDirs(cacheDir); + + string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false); + + Path dataFile = cacheDir + "/" + urlHash + ".info"; + Path fileLink = cacheDir + "/" + urlHash + "-file"; + + PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink)); + + Path storePath; + + string expectedETag; + + bool skip = false; + + CachedDownloadResult result; + + if (pathExists(fileLink) && pathExists(dataFile)) { + storePath = readLink(fileLink); + store->addTempRoot(storePath); + if (store->isValidPath(storePath)) { + auto ss = tokenizeString>(readFile(dataFile), "\n"); + if (ss.size() >= 3 && ss[0] == url) { + time_t lastChecked; + if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) { + skip = true; + result.effectiveUri = request.uri; + result.etag = ss[1]; + } else if (!ss[1].empty()) { + debug(format("verifying previous ETag '%1%'") % ss[1]); + expectedETag = ss[1]; + } + } + } else + storePath = ""; + } + + if (!skip) { + + try { + DownloadRequest request2(url); + request2.expectedETag = expectedETag; + auto res = download(request2); + result.effectiveUri = res.effectiveUri; + result.etag = res.etag; + + if (!res.cached) { + ValidPathInfo info; + StringSink sink; + dumpString(*res.data, sink); + Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data); + info.path = store->makeFixedOutputPath(false, hash, name); + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(false, hash); + store->addToStore(info, sink.s, NoRepair, NoCheckSigs); + storePath = info.path; + } + + assert(!storePath.empty()); + replaceSymlink(storePath, fileLink); + + writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n"); + } catch (DownloadError & e) { + if (storePath.empty()) throw; + warn("warning: %s; using cached result", e.msg()); + result.etag = expectedETag; + } + } + + if (request.unpack) { + Path unpackedLink = cacheDir + "/" + baseNameOf(storePath) + "-unpacked"; + PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink)); + Path unpackedStorePath; + if (pathExists(unpackedLink)) { + unpackedStorePath = readLink(unpackedLink); + store->addTempRoot(unpackedStorePath); + if (!store->isValidPath(unpackedStorePath)) + unpackedStorePath = ""; + } + if (unpackedStorePath.empty()) { + printInfo(format("unpacking '%1%'...") % url); + Path tmpDir = createTempDir(); + AutoDelete autoDelete(tmpDir, true); + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", store->toRealPath(storePath), "-C", tmpDir, "--strip-components", "1"}); + unpackedStorePath = store->addToStore(name, tmpDir, true, htSHA256, defaultPathFilter, NoRepair); + } + replaceSymlink(unpackedStorePath, unpackedLink); + storePath = unpackedStorePath; + } + + if (expectedStorePath != "" && storePath != expectedStorePath) { + unsigned int statusCode = 102; + Hash gotHash = request.unpack + ? hashPath(request.expectedHash.type, store->toRealPath(storePath)).first + : hashFile(request.expectedHash.type, store->toRealPath(storePath)); + throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s", + url, request.expectedHash.to_string(), gotHash.to_string()); + } + + result.storePath = storePath; + result.path = store->toRealPath(storePath); + return result; +} + + +bool isUri(const string & s) +{ + if (s.compare(0, 8, "channel:") == 0) return true; + size_t pos = s.find("://"); + if (pos == string::npos) return false; + string scheme(s, 0, pos); + return scheme == "http" || scheme == "https" || scheme == "file" || scheme == "channel" || scheme == "git" || scheme == "s3" || scheme == "ssh"; +} + + +} diff --git a/third_party/nix/src/libstore/download.hh b/third_party/nix/src/libstore/download.hh new file mode 100644 index 000000000..68565bf46 --- /dev/null +++ b/third_party/nix/src/libstore/download.hh @@ -0,0 +1,138 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "globals.hh" + +#include +#include + +namespace nix { + +struct DownloadSettings : Config +{ + Setting enableHttp2{this, true, "http2", + "Whether to enable HTTP/2 support."}; + + Setting userAgentSuffix{this, "", "user-agent-suffix", + "String appended to the user agent in HTTP requests."}; + + Setting httpConnections{this, 25, "http-connections", + "Number of parallel HTTP connections.", + {"binary-caches-parallel-connections"}}; + + Setting connectTimeout{this, 0, "connect-timeout", + "Timeout for connecting to servers during downloads. 0 means use curl's builtin default."}; + + Setting stalledDownloadTimeout{this, 300, "stalled-download-timeout", + "Timeout (in seconds) for receiving data from servers during download. Nix cancels idle downloads after this timeout's duration."}; + + Setting tries{this, 5, "download-attempts", + "How often Nix will attempt to download a file before giving up."}; +}; + +extern DownloadSettings downloadSettings; + +struct DownloadRequest +{ + std::string uri; + std::string expectedETag; + bool verifyTLS = true; + bool head = false; + size_t tries = downloadSettings.tries; + unsigned int baseRetryTimeMs = 250; + ActivityId parentAct; + bool decompress = true; + std::shared_ptr data; + std::string mimeType; + std::function dataCallback; + + DownloadRequest(const std::string & uri) + : uri(uri), parentAct(getCurActivity()) { } + + std::string verb() + { + return data ? "upload" : "download"; + } +}; + +struct DownloadResult +{ + bool cached = false; + std::string etag; + std::string effectiveUri; + std::shared_ptr data; + uint64_t bodySize = 0; +}; + +struct CachedDownloadRequest +{ + std::string uri; + bool unpack = false; + std::string name; + Hash expectedHash; + unsigned int ttl = settings.tarballTtl; + + CachedDownloadRequest(const std::string & uri) + : uri(uri) { } +}; + +struct CachedDownloadResult +{ + // Note: 'storePath' may be different from 'path' when using a + // chroot store. + Path storePath; + Path path; + std::optional etag; + std::string effectiveUri; +}; + +class Store; + +struct Downloader +{ + virtual ~Downloader() { } + + /* Enqueue a download request, returning a future to the result of + the download. The future may throw a DownloadError + exception. */ + virtual void enqueueDownload(const DownloadRequest & request, + Callback callback) = 0; + + std::future enqueueDownload(const DownloadRequest & request); + + /* Synchronously download a file. */ + DownloadResult download(const DownloadRequest & request); + + /* Download a file, writing its data to a sink. The sink will be + invoked on the thread of the caller. */ + void download(DownloadRequest && request, Sink & sink); + + /* Check if the specified file is already in ~/.cache/nix/tarballs + and is more recent than ‘tarball-ttl’ seconds. Otherwise, + use the recorded ETag to verify if the server has a more + recent version, and if so, download it to the Nix store. */ + CachedDownloadResult downloadCached(ref store, const CachedDownloadRequest & request); + + enum Error { NotFound, Forbidden, Misc, Transient, Interrupted }; +}; + +/* Return a shared Downloader object. Using this object is preferred + because it enables connection reuse and HTTP/2 multiplexing. */ +ref getDownloader(); + +/* Return a new Downloader object. */ +ref makeDownloader(); + +class DownloadError : public Error +{ +public: + Downloader::Error error; + DownloadError(Downloader::Error error, const FormatOrString & fs) + : Error(fs), error(error) + { } +}; + +bool isUri(const string & s); + +} diff --git a/third_party/nix/src/libstore/export-import.cc b/third_party/nix/src/libstore/export-import.cc new file mode 100644 index 000000000..ef3fea7c8 --- /dev/null +++ b/third_party/nix/src/libstore/export-import.cc @@ -0,0 +1,106 @@ +#include "store-api.hh" +#include "archive.hh" +#include "worker-protocol.hh" + +#include + +namespace nix { + +struct HashAndWriteSink : Sink +{ + Sink & writeSink; + HashSink hashSink; + HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) + { + } + virtual void operator () (const unsigned char * data, size_t len) + { + writeSink(data, len); + hashSink(data, len); + } + Hash currentHash() + { + return hashSink.currentHash().first; + } +}; + +void Store::exportPaths(const Paths & paths, Sink & sink) +{ + Paths sorted = topoSortPaths(PathSet(paths.begin(), paths.end())); + std::reverse(sorted.begin(), sorted.end()); + + std::string doneLabel("paths exported"); + //logger->incExpected(doneLabel, sorted.size()); + + for (auto & path : sorted) { + //Activity act(*logger, lvlInfo, format("exporting path '%s'") % path); + sink << 1; + exportPath(path, sink); + //logger->incProgress(doneLabel); + } + + sink << 0; +} + +void Store::exportPath(const Path & path, Sink & sink) +{ + auto info = queryPathInfo(path); + + HashAndWriteSink hashAndWriteSink(sink); + + narFromPath(path, hashAndWriteSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashAndWriteSink.currentHash(); + if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + throw Error(format("hash of path '%1%' has changed from '%2%' to '%3%'!") % path + % info->narHash.to_string() % hash.to_string()); + + hashAndWriteSink << exportMagic << path << info->references << info->deriver << 0; +} + +Paths Store::importPaths(Source & source, std::shared_ptr accessor, CheckSigsFlag checkSigs) +{ + Paths res; + while (true) { + auto n = readNum(source); + if (n == 0) break; + if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'"); + + /* Extract the NAR from the source. */ + TeeSink tee(source); + parseDump(tee, tee.source); + + uint32_t magic = readInt(source); + if (magic != exportMagic) + throw Error("Nix archive cannot be imported; wrong format"); + + ValidPathInfo info; + + info.path = readStorePath(*this, source); + + //Activity act(*logger, lvlInfo, format("importing path '%s'") % info.path); + + info.references = readStorePaths(*this, source); + + info.deriver = readString(source); + if (info.deriver != "") assertStorePath(info.deriver); + + info.narHash = hashString(htSHA256, *tee.source.data); + info.narSize = tee.source.data->size(); + + // Ignore optional legacy signature. + if (readInt(source) == 1) + readString(source); + + addToStore(info, tee.source.data, NoRepair, checkSigs, accessor); + + res.push_back(info.path); + } + + return res; +} + +} diff --git a/third_party/nix/src/libstore/fs-accessor.hh b/third_party/nix/src/libstore/fs-accessor.hh new file mode 100644 index 000000000..64780a6da --- /dev/null +++ b/third_party/nix/src/libstore/fs-accessor.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +/* An abstract class for accessing a filesystem-like structure, such + as a (possibly remote) Nix store or the contents of a NAR file. */ +class FSAccessor +{ +public: + enum Type { tMissing, tRegular, tSymlink, tDirectory }; + + struct Stat + { + Type type = tMissing; + uint64_t fileSize = 0; // regular files only + bool isExecutable = false; // regular files only + uint64_t narOffset = 0; // regular files only + }; + + virtual ~FSAccessor() { } + + virtual Stat stat(const Path & path) = 0; + + virtual StringSet readDirectory(const Path & path) = 0; + + virtual std::string readFile(const Path & path) = 0; + + virtual std::string readLink(const Path & path) = 0; +}; + +} diff --git a/third_party/nix/src/libstore/gc.cc b/third_party/nix/src/libstore/gc.cc new file mode 100644 index 000000000..a166f4ee2 --- /dev/null +++ b/third_party/nix/src/libstore/gc.cc @@ -0,0 +1,948 @@ +#include "derivations.hh" +#include "globals.hh" +#include "local-store.hh" +#include "finally.hh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace nix { + + +static string gcLockName = "gc.lock"; +static string gcRootsDir = "gcroots"; + + +/* Acquire the global GC lock. This is used to prevent new Nix + processes from starting after the temporary root files have been + read. To be precise: when they try to create a new temporary root + file, they will block until the garbage collector has finished / + yielded the GC lock. */ +AutoCloseFD LocalStore::openGCLock(LockType lockType) +{ + Path fnGCLock = (format("%1%/%2%") + % stateDir % gcLockName).str(); + + debug(format("acquiring global GC lock '%1%'") % fnGCLock); + + AutoCloseFD fdGCLock = open(fnGCLock.c_str(), O_RDWR | O_CREAT | O_CLOEXEC, 0600); + if (!fdGCLock) + throw SysError(format("opening global GC lock '%1%'") % fnGCLock); + + if (!lockFile(fdGCLock.get(), lockType, false)) { + printError(format("waiting for the big garbage collector lock...")); + lockFile(fdGCLock.get(), lockType, true); + } + + /* !!! Restrict read permission on the GC root. Otherwise any + process that can open the file for reading can DoS the + collector. */ + + return fdGCLock; +} + + +static void makeSymlink(const Path & link, const Path & target) +{ + /* Create directories up to `gcRoot'. */ + createDirs(dirOf(link)); + + /* Create the new symlink. */ + Path tempLink = (format("%1%.tmp-%2%-%3%") + % link % getpid() % random()).str(); + createSymlink(target, tempLink); + + /* Atomically replace the old one. */ + if (rename(tempLink.c_str(), link.c_str()) == -1) + throw SysError(format("cannot rename '%1%' to '%2%'") + % tempLink % link); +} + + +void LocalStore::syncWithGC() +{ + AutoCloseFD fdGCLock = openGCLock(ltRead); +} + + +void LocalStore::addIndirectRoot(const Path & path) +{ + string hash = hashString(htSHA1, path).to_string(Base32, false); + Path realRoot = canonPath((format("%1%/%2%/auto/%3%") + % stateDir % gcRootsDir % hash).str()); + makeSymlink(realRoot, path); +} + + +Path LocalFSStore::addPermRoot(const Path & _storePath, + const Path & _gcRoot, bool indirect, bool allowOutsideRootsDir) +{ + Path storePath(canonPath(_storePath)); + Path gcRoot(canonPath(_gcRoot)); + assertStorePath(storePath); + + if (isInStore(gcRoot)) + throw Error(format( + "creating a garbage collector root (%1%) in the Nix store is forbidden " + "(are you running nix-build inside the store?)") % gcRoot); + + if (indirect) { + /* Don't clobber the link if it already exists and doesn't + point to the Nix store. */ + if (pathExists(gcRoot) && (!isLink(gcRoot) || !isInStore(readLink(gcRoot)))) + throw Error(format("cannot create symlink '%1%'; already exists") % gcRoot); + makeSymlink(gcRoot, storePath); + addIndirectRoot(gcRoot); + } + + else { + if (!allowOutsideRootsDir) { + Path rootsDir = canonPath((format("%1%/%2%") % stateDir % gcRootsDir).str()); + + if (string(gcRoot, 0, rootsDir.size() + 1) != rootsDir + "/") + throw Error(format( + "path '%1%' is not a valid garbage collector root; " + "it's not in the directory '%2%'") + % gcRoot % rootsDir); + } + + if (baseNameOf(gcRoot) == baseNameOf(storePath)) + writeFile(gcRoot, ""); + else + makeSymlink(gcRoot, storePath); + } + + /* Check that the root can be found by the garbage collector. + !!! This can be very slow on machines that have many roots. + Instead of reading all the roots, it would be more efficient to + check if the root is in a directory in or linked from the + gcroots directory. */ + if (settings.checkRootReachability) { + Roots roots = findRoots(false); + if (roots[storePath].count(gcRoot) == 0) + printError( + format( + "warning: '%1%' is not in a directory where the garbage collector looks for roots; " + "therefore, '%2%' might be removed by the garbage collector") + % gcRoot % storePath); + } + + /* Grab the global GC root, causing us to block while a GC is in + progress. This prevents the set of permanent roots from + increasing while a GC is in progress. */ + syncWithGC(); + + return gcRoot; +} + + +void LocalStore::addTempRoot(const Path & path) +{ + auto state(_state.lock()); + + /* Create the temporary roots file for this process. */ + if (!state->fdTempRoots) { + + while (1) { + AutoCloseFD fdGCLock = openGCLock(ltRead); + + if (pathExists(fnTempRoots)) + /* It *must* be stale, since there can be no two + processes with the same pid. */ + unlink(fnTempRoots.c_str()); + + state->fdTempRoots = openLockFile(fnTempRoots, true); + + fdGCLock = -1; + + debug(format("acquiring read lock on '%1%'") % fnTempRoots); + lockFile(state->fdTempRoots.get(), ltRead, true); + + /* Check whether the garbage collector didn't get in our + way. */ + struct stat st; + if (fstat(state->fdTempRoots.get(), &st) == -1) + throw SysError(format("statting '%1%'") % fnTempRoots); + if (st.st_size == 0) break; + + /* The garbage collector deleted this file before we could + get a lock. (It won't delete the file after we get a + lock.) Try again. */ + } + + } + + /* Upgrade the lock to a write lock. This will cause us to block + if the garbage collector is holding our lock. */ + debug(format("acquiring write lock on '%1%'") % fnTempRoots); + lockFile(state->fdTempRoots.get(), ltWrite, true); + + string s = path + '\0'; + writeFull(state->fdTempRoots.get(), s); + + /* Downgrade to a read lock. */ + debug(format("downgrading to read lock on '%1%'") % fnTempRoots); + lockFile(state->fdTempRoots.get(), ltRead, true); +} + + +static std::string censored = "{censored}"; + + +void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor) +{ + /* Read the `temproots' directory for per-process temporary root + files. */ + for (auto & i : readDirectory(tempRootsDir)) { + Path path = tempRootsDir + "/" + i.name; + + pid_t pid = std::stoi(i.name); + + debug(format("reading temporary root file '%1%'") % path); + FDPtr fd(new AutoCloseFD(open(path.c_str(), O_CLOEXEC | O_RDWR, 0666))); + if (!*fd) { + /* It's okay if the file has disappeared. */ + if (errno == ENOENT) continue; + throw SysError(format("opening temporary roots file '%1%'") % path); + } + + /* This should work, but doesn't, for some reason. */ + //FDPtr fd(new AutoCloseFD(openLockFile(path, false))); + //if (*fd == -1) continue; + + /* Try to acquire a write lock without blocking. This can + only succeed if the owning process has died. In that case + we don't care about its temporary roots. */ + if (lockFile(fd->get(), ltWrite, false)) { + printError(format("removing stale temporary roots file '%1%'") % path); + unlink(path.c_str()); + writeFull(fd->get(), "d"); + continue; + } + + /* Acquire a read lock. This will prevent the owning process + from upgrading to a write lock, therefore it will block in + addTempRoot(). */ + debug(format("waiting for read lock on '%1%'") % path); + lockFile(fd->get(), ltRead, true); + + /* Read the entire file. */ + string contents = readFile(fd->get()); + + /* Extract the roots. */ + string::size_type pos = 0, end; + + while ((end = contents.find((char) 0, pos)) != string::npos) { + Path root(contents, pos, end - pos); + debug("got temporary root '%s'", root); + assertStorePath(root); + tempRoots[root].emplace(censor ? censored : fmt("{temp:%d}", pid)); + pos = end + 1; + } + + fds.push_back(fd); /* keep open */ + } +} + + +void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots) +{ + auto foundRoot = [&](const Path & path, const Path & target) { + Path storePath = toStorePath(target); + if (isStorePath(storePath) && isValidPath(storePath)) + roots[storePath].emplace(path); + else + printInfo(format("skipping invalid root from '%1%' to '%2%'") % path % storePath); + }; + + try { + + if (type == DT_UNKNOWN) + type = getFileType(path); + + if (type == DT_DIR) { + for (auto & i : readDirectory(path)) + findRoots(path + "/" + i.name, i.type, roots); + } + + else if (type == DT_LNK) { + Path target = readLink(path); + if (isInStore(target)) + foundRoot(path, target); + + /* Handle indirect roots. */ + else { + target = absPath(target, dirOf(path)); + if (!pathExists(target)) { + if (isInDir(path, stateDir + "/" + gcRootsDir + "/auto")) { + printInfo(format("removing stale link from '%1%' to '%2%'") % path % target); + unlink(path.c_str()); + } + } else { + struct stat st2 = lstat(target); + if (!S_ISLNK(st2.st_mode)) return; + Path target2 = readLink(target); + if (isInStore(target2)) foundRoot(target, target2); + } + } + } + + else if (type == DT_REG) { + Path storePath = storeDir + "/" + baseNameOf(path); + if (isStorePath(storePath) && isValidPath(storePath)) + roots[storePath].emplace(path); + } + + } + + catch (SysError & e) { + /* We only ignore permanent failures. */ + if (e.errNo == EACCES || e.errNo == ENOENT || e.errNo == ENOTDIR) + printInfo(format("cannot read potential root '%1%'") % path); + else + throw; + } +} + + +void LocalStore::findRootsNoTemp(Roots & roots, bool censor) +{ + /* Process direct roots in {gcroots,profiles}. */ + findRoots(stateDir + "/" + gcRootsDir, DT_UNKNOWN, roots); + findRoots(stateDir + "/profiles", DT_UNKNOWN, roots); + + /* Add additional roots returned by different platforms-specific + heuristics. This is typically used to add running programs to + the set of roots (to prevent them from being garbage collected). */ + findRuntimeRoots(roots, censor); +} + + +Roots LocalStore::findRoots(bool censor) +{ + Roots roots; + findRootsNoTemp(roots, censor); + + FDs fds; + findTempRoots(fds, roots, censor); + + return roots; +} + +static void readProcLink(const string & file, Roots & roots) +{ + /* 64 is the starting buffer size gnu readlink uses... */ + auto bufsiz = ssize_t{64}; +try_again: + char buf[bufsiz]; + auto res = readlink(file.c_str(), buf, bufsiz); + if (res == -1) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) + return; + throw SysError("reading symlink"); + } + if (res == bufsiz) { + if (SSIZE_MAX / 2 < bufsiz) + throw Error("stupidly long symlink"); + bufsiz *= 2; + goto try_again; + } + if (res > 0 && buf[0] == '/') + roots[std::string(static_cast(buf), res)] + .emplace(file); +} + +static string quoteRegexChars(const string & raw) +{ + static auto specialRegex = std::regex(R"([.^$\\*+?()\[\]{}|])"); + return std::regex_replace(raw, specialRegex, R"(\$&)"); +} + +static void readFileRoots(const char * path, Roots & roots) +{ + try { + roots[readFile(path)].emplace(path); + } catch (SysError & e) { + if (e.errNo != ENOENT && e.errNo != EACCES) + throw; + } +} + +void LocalStore::findRuntimeRoots(Roots & roots, bool censor) +{ + Roots unchecked; + + auto procDir = AutoCloseDir{opendir("/proc")}; + if (procDir) { + struct dirent * ent; + auto digitsRegex = std::regex(R"(^\d+$)"); + auto mapRegex = std::regex(R"(^\s*\S+\s+\S+\s+\S+\s+\S+\s+\S+\s+(/\S+)\s*$)"); + auto storePathRegex = std::regex(quoteRegexChars(storeDir) + R"(/[0-9a-z]+[0-9a-zA-Z\+\-\._\?=]*)"); + while (errno = 0, ent = readdir(procDir.get())) { + checkInterrupt(); + if (std::regex_match(ent->d_name, digitsRegex)) { + readProcLink(fmt("/proc/%s/exe" ,ent->d_name), unchecked); + readProcLink(fmt("/proc/%s/cwd", ent->d_name), unchecked); + + auto fdStr = fmt("/proc/%s/fd", ent->d_name); + auto fdDir = AutoCloseDir(opendir(fdStr.c_str())); + if (!fdDir) { + if (errno == ENOENT || errno == EACCES) + continue; + throw SysError(format("opening %1%") % fdStr); + } + struct dirent * fd_ent; + while (errno = 0, fd_ent = readdir(fdDir.get())) { + if (fd_ent->d_name[0] != '.') + readProcLink(fmt("%s/%s", fdStr, fd_ent->d_name), unchecked); + } + if (errno) { + if (errno == ESRCH) + continue; + throw SysError(format("iterating /proc/%1%/fd") % ent->d_name); + } + fdDir.reset(); + + try { + auto mapFile = fmt("/proc/%s/maps", ent->d_name); + auto mapLines = tokenizeString>(readFile(mapFile, true), "\n"); + for (const auto & line : mapLines) { + auto match = std::smatch{}; + if (std::regex_match(line, match, mapRegex)) + unchecked[match[1]].emplace(mapFile); + } + + auto envFile = fmt("/proc/%s/environ", ent->d_name); + auto envString = readFile(envFile, true); + auto env_end = std::sregex_iterator{}; + for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i) + unchecked[i->str()].emplace(envFile); + } catch (SysError & e) { + if (errno == ENOENT || errno == EACCES || errno == ESRCH) + continue; + throw; + } + } + } + if (errno) + throw SysError("iterating /proc"); + } + +#if !defined(__linux__) + // lsof is really slow on OS X. This actually causes the gc-concurrent.sh test to fail. + // See: https://github.com/NixOS/nix/issues/3011 + // Because of this we disable lsof when running the tests. + if (getEnv("_NIX_TEST_NO_LSOF") == "") { + try { + std::regex lsofRegex(R"(^n(/.*)$)"); + auto lsofLines = + tokenizeString>(runProgram(LSOF, true, { "-n", "-w", "-F", "n" }), "\n"); + for (const auto & line : lsofLines) { + std::smatch match; + if (std::regex_match(line, match, lsofRegex)) + unchecked[match[1]].emplace("{lsof}"); + } + } catch (ExecError & e) { + /* lsof not installed, lsof failed */ + } + } +#endif + +#if defined(__linux__) + readFileRoots("/proc/sys/kernel/modprobe", unchecked); + readFileRoots("/proc/sys/kernel/fbsplash", unchecked); + readFileRoots("/proc/sys/kernel/poweroff_cmd", unchecked); +#endif + + for (auto & [target, links] : unchecked) { + if (isInStore(target)) { + Path path = toStorePath(target); + if (isStorePath(path) && isValidPath(path)) { + debug(format("got additional root '%1%'") % path); + if (censor) + roots[path].insert(censored); + else + roots[path].insert(links.begin(), links.end()); + } + } + } +} + + +struct GCLimitReached { }; + + +struct LocalStore::GCState +{ + GCOptions options; + GCResults & results; + PathSet roots; + PathSet tempRoots; + PathSet dead; + PathSet alive; + bool gcKeepOutputs; + bool gcKeepDerivations; + unsigned long long bytesInvalidated; + bool moveToTrash = true; + bool shouldDelete; + GCState(GCResults & results_) : results(results_), bytesInvalidated(0) { } +}; + + +bool LocalStore::isActiveTempFile(const GCState & state, + const Path & path, const string & suffix) +{ + return hasSuffix(path, suffix) + && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); +} + + +void LocalStore::deleteGarbage(GCState & state, const Path & path) +{ + unsigned long long bytesFreed; + deletePath(path, bytesFreed); + state.results.bytesFreed += bytesFreed; +} + + +void LocalStore::deletePathRecursive(GCState & state, const Path & path) +{ + checkInterrupt(); + + unsigned long long size = 0; + + if (isStorePath(path) && isValidPath(path)) { + PathSet referrers; + queryReferrers(path, referrers); + for (auto & i : referrers) + if (i != path) deletePathRecursive(state, i); + size = queryPathInfo(path)->narSize; + invalidatePathChecked(path); + } + + Path realPath = realStoreDir + "/" + baseNameOf(path); + + struct stat st; + if (lstat(realPath.c_str(), &st)) { + if (errno == ENOENT) return; + throw SysError(format("getting status of %1%") % realPath); + } + + printInfo(format("deleting '%1%'") % path); + + state.results.paths.insert(path); + + /* If the path is not a regular file or symlink, move it to the + trash directory. The move is to ensure that later (when we're + not holding the global GC lock) we can delete the path without + being afraid that the path has become alive again. Otherwise + delete it right away. */ + if (state.moveToTrash && S_ISDIR(st.st_mode)) { + // Estimate the amount freed using the narSize field. FIXME: + // if the path was not valid, need to determine the actual + // size. + try { + if (chmod(realPath.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("making '%1%' writable") % realPath); + Path tmp = trashDir + "/" + baseNameOf(path); + if (rename(realPath.c_str(), tmp.c_str())) + throw SysError(format("unable to rename '%1%' to '%2%'") % realPath % tmp); + state.bytesInvalidated += size; + } catch (SysError & e) { + if (e.errNo == ENOSPC) { + printInfo(format("note: can't create move '%1%': %2%") % realPath % e.msg()); + deleteGarbage(state, realPath); + } + } + } else + deleteGarbage(state, realPath); + + if (state.results.bytesFreed + state.bytesInvalidated > state.options.maxFreed) { + printInfo(format("deleted or invalidated more than %1% bytes; stopping") % state.options.maxFreed); + throw GCLimitReached(); + } +} + + +bool LocalStore::canReachRoot(GCState & state, PathSet & visited, const Path & path) +{ + if (visited.count(path)) return false; + + if (state.alive.count(path)) return true; + + if (state.dead.count(path)) return false; + + if (state.roots.count(path)) { + debug(format("cannot delete '%1%' because it's a root") % path); + state.alive.insert(path); + return true; + } + + visited.insert(path); + + if (!isStorePath(path) || !isValidPath(path)) return false; + + PathSet incoming; + + /* Don't delete this path if any of its referrers are alive. */ + queryReferrers(path, incoming); + + /* If keep-derivations is set and this is a derivation, then + don't delete the derivation if any of the outputs are alive. */ + if (state.gcKeepDerivations && isDerivation(path)) { + PathSet outputs = queryDerivationOutputs(path); + for (auto & i : outputs) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + incoming.insert(i); + } + + /* If keep-outputs is set, then don't delete this path if there + are derivers of this path that are not garbage. */ + if (state.gcKeepOutputs) { + PathSet derivers = queryValidDerivers(path); + for (auto & i : derivers) + incoming.insert(i); + } + + for (auto & i : incoming) + if (i != path) + if (canReachRoot(state, visited, i)) { + state.alive.insert(path); + return true; + } + + return false; +} + + +void LocalStore::tryToDelete(GCState & state, const Path & path) +{ + checkInterrupt(); + + auto realPath = realStoreDir + "/" + baseNameOf(path); + if (realPath == linksDir || realPath == trashDir) return; + + //Activity act(*logger, lvlDebug, format("considering whether to delete '%1%'") % path); + + if (!isStorePath(path) || !isValidPath(path)) { + /* A lock file belonging to a path that we're building right + now isn't garbage. */ + if (isActiveTempFile(state, path, ".lock")) return; + + /* Don't delete .chroot directories for derivations that are + currently being built. */ + if (isActiveTempFile(state, path, ".chroot")) return; + + /* Don't delete .check directories for derivations that are + currently being built, because we may need to run + diff-hook. */ + if (isActiveTempFile(state, path, ".check")) return; + } + + PathSet visited; + + if (canReachRoot(state, visited, path)) { + debug(format("cannot delete '%1%' because it's still reachable") % path); + } else { + /* No path we visited was a root, so everything is garbage. + But we only delete ‘path’ and its referrers here so that + ‘nix-store --delete’ doesn't have the unexpected effect of + recursing into derivations and outputs. */ + state.dead.insert(visited.begin(), visited.end()); + if (state.shouldDelete) + deletePathRecursive(state, path); + } +} + + +/* Unlink all files in /nix/store/.links that have a link count of 1, + which indicates that there are no other links and so they can be + safely deleted. FIXME: race condition with optimisePath(): we + might see a link count of 1 just before optimisePath() increases + the link count. */ +void LocalStore::removeUnusedLinks(const GCState & state) +{ + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + + long long actualSize = 0, unsharedSize = 0; + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = linksDir + "/" + name; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) + throw SysError(format("statting '%1%'") % path); + + if (st.st_nlink != 1) { + actualSize += st.st_size; + unsharedSize += (st.st_nlink - 1) * st.st_size; + continue; + } + + printMsg(lvlTalkative, format("deleting unused link '%1%'") % path); + + if (unlink(path.c_str()) == -1) + throw SysError(format("deleting '%1%'") % path); + + state.results.bytesFreed += st.st_size; + } + + struct stat st; + if (stat(linksDir.c_str(), &st) == -1) + throw SysError(format("statting '%1%'") % linksDir); + long long overhead = st.st_blocks * 512ULL; + + printInfo(format("note: currently hard linking saves %.2f MiB") + % ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0))); +} + + +void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + GCState state(results); + state.options = options; + state.gcKeepOutputs = settings.gcKeepOutputs; + state.gcKeepDerivations = settings.gcKeepDerivations; + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `keep-outputs' or `keep-derivations' are true + (the garbage collector will recurse into deleting the outputs + or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } + + state.shouldDelete = options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific; + + if (state.shouldDelete) + deletePath(reservedPath); + + /* Acquire the global GC root. This prevents + a) New roots from being added. + b) Processes from creating new temporary root files. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + /* Find the roots. Since we've grabbed the GC lock, the set of + permanent roots cannot increase now. */ + printError(format("finding garbage collector roots...")); + Roots rootMap; + if (!options.ignoreLiveness) + findRootsNoTemp(rootMap, true); + + for (auto & i : rootMap) state.roots.insert(i.first); + + /* Read the temporary roots. This acquires read locks on all + per-process temporary root files. So after this point no paths + can be added to the set of temporary roots. */ + FDs fds; + Roots tempRoots; + findTempRoots(fds, tempRoots, true); + for (auto & root : tempRoots) + state.tempRoots.insert(root.first); + state.roots.insert(state.tempRoots.begin(), state.tempRoots.end()); + + /* After this point the set of roots or temporary roots cannot + increase, since we hold locks on everything. So everything + that is not reachable from `roots' is garbage. */ + + if (state.shouldDelete) { + if (pathExists(trashDir)) deleteGarbage(state, trashDir); + try { + createDirs(trashDir); + } catch (SysError & e) { + if (e.errNo == ENOSPC) { + printInfo(format("note: can't create trash directory: %1%") % e.msg()); + state.moveToTrash = false; + } + } + } + + /* Now either delete all garbage paths, or just the specified + paths (for gcDeleteSpecific). */ + + if (options.action == GCOptions::gcDeleteSpecific) { + + for (auto & i : options.pathsToDelete) { + assertStorePath(i); + tryToDelete(state, i); + if (state.dead.find(i) == state.dead.end()) + throw Error(format("cannot delete path '%1%' since it is still alive") % i); + } + + } else if (options.maxFreed > 0) { + + if (state.shouldDelete) + printError(format("deleting garbage...")); + else + printError(format("determining live/dead paths...")); + + try { + + AutoCloseDir dir(opendir(realStoreDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % realStoreDir); + + /* Read the store and immediately delete all paths that + aren't valid. When using --max-freed etc., deleting + invalid paths is preferred over deleting unreachable + paths, since unreachable paths could become reachable + again. We don't use readDirectory() here so that GCing + can start faster. */ + Paths entries; + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + Path path = storeDir + "/" + name; + if (isStorePath(path) && isValidPath(path)) + entries.push_back(path); + else + tryToDelete(state, path); + } + + dir.reset(); + + /* Now delete the unreachable valid paths. Randomise the + order in which we delete entries to make the collector + less biased towards deleting paths that come + alphabetically first (e.g. /nix/store/000...). This + matters when using --max-freed etc. */ + vector entries_(entries.begin(), entries.end()); + std::mt19937 gen(1); + std::shuffle(entries_.begin(), entries_.end(), gen); + + for (auto & i : entries_) + tryToDelete(state, i); + + } catch (GCLimitReached & e) { + } + } + + if (state.options.action == GCOptions::gcReturnLive) { + state.results.paths = state.alive; + return; + } + + if (state.options.action == GCOptions::gcReturnDead) { + state.results.paths = state.dead; + return; + } + + /* Allow other processes to add to the store from here on. */ + fdGCLock = -1; + fds.clear(); + + /* Delete the trash directory. */ + printInfo(format("deleting '%1%'") % trashDir); + deleteGarbage(state, trashDir); + + /* Clean up the links directory. */ + if (options.action == GCOptions::gcDeleteDead || options.action == GCOptions::gcDeleteSpecific) { + printError(format("deleting unused links...")); + removeUnusedLinks(state); + } + + /* While we're at it, vacuum the database. */ + //if (options.action == GCOptions::gcDeleteDead) vacuumDB(); +} + + +void LocalStore::autoGC(bool sync) +{ + static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE", ""); + + auto getAvail = [this]() -> uint64_t { + if (!fakeFreeSpaceFile.empty()) + return std::stoll(readFile(fakeFreeSpaceFile)); + + struct statvfs st; + if (statvfs(realStoreDir.c_str(), &st)) + throw SysError("getting filesystem info about '%s'", realStoreDir); + + return (uint64_t) st.f_bavail * st.f_bsize; + }; + + std::shared_future future; + + { + auto state(_state.lock()); + + if (state->gcRunning) { + future = state->gcFuture; + debug("waiting for auto-GC to finish"); + goto sync; + } + + auto now = std::chrono::steady_clock::now(); + + if (now < state->lastGCCheck + std::chrono::seconds(settings.minFreeCheckInterval)) return; + + auto avail = getAvail(); + + state->lastGCCheck = now; + + if (avail >= settings.minFree || avail >= settings.maxFree) return; + + if (avail > state->availAfterGC * 0.97) return; + + state->gcRunning = true; + + std::promise promise; + future = state->gcFuture = promise.get_future().share(); + + std::thread([promise{std::move(promise)}, this, avail, getAvail]() mutable { + + try { + + /* Wake up any threads waiting for the auto-GC to finish. */ + Finally wakeup([&]() { + auto state(_state.lock()); + state->gcRunning = false; + state->lastGCCheck = std::chrono::steady_clock::now(); + promise.set_value(); + }); + + GCOptions options; + options.maxFreed = settings.maxFree - avail; + + printInfo("running auto-GC to free %d bytes", options.maxFreed); + + GCResults results; + + collectGarbage(options, results); + + _state.lock()->availAfterGC = getAvail(); + + } catch (...) { + // FIXME: we could propagate the exception to the + // future, but we don't really care. + ignoreException(); + } + + }).detach(); + } + + sync: + // Wait for the future outside of the state lock. + if (sync) future.get(); +} + + +} diff --git a/third_party/nix/src/libstore/globals.cc b/third_party/nix/src/libstore/globals.cc new file mode 100644 index 000000000..1c2c08715 --- /dev/null +++ b/third_party/nix/src/libstore/globals.cc @@ -0,0 +1,187 @@ +#include "globals.hh" +#include "util.hh" +#include "archive.hh" +#include "args.hh" + +#include +#include +#include +#include + + +namespace nix { + + +/* The default location of the daemon socket, relative to nixStateDir. + The socket is in a directory to allow you to control access to the + Nix daemon by setting the mode/ownership of the directory + appropriately. (This wouldn't work on the socket itself since it + must be deleted and recreated on startup.) */ +#define DEFAULT_SOCKET_PATH "/daemon-socket/socket" + +/* chroot-like behavior from Apple's sandbox */ +#if __APPLE__ + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "/System/Library /usr/lib /dev /bin/sh" +#else + #define DEFAULT_ALLOWED_IMPURE_PREFIXES "" +#endif + +Settings settings; + +static GlobalConfig::Register r1(&settings); + +Settings::Settings() + : nixPrefix(NIX_PREFIX) + , nixStore(canonPath(getEnv("NIX_STORE_DIR", getEnv("NIX_STORE", NIX_STORE_DIR)))) + , nixDataDir(canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR))) + , nixLogDir(canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR))) + , nixStateDir(canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR))) + , nixConfDir(canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR))) + , nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR))) + , nixBinDir(canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR))) + , nixManDir(canonPath(NIX_MAN_DIR)) + , nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH)) +{ + buildUsersGroup = getuid() == 0 ? "nixbld" : ""; + lockCPU = getEnv("NIX_AFFINITY_HACK", "1") == "1"; + + caFile = getEnv("NIX_SSL_CERT_FILE", getEnv("SSL_CERT_FILE", "")); + if (caFile == "") { + for (auto & fn : {"/etc/ssl/certs/ca-certificates.crt", "/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt"}) + if (pathExists(fn)) { + caFile = fn; + break; + } + } + + /* Backwards compatibility. */ + auto s = getEnv("NIX_REMOTE_SYSTEMS"); + if (s != "") { + Strings ss; + for (auto & p : tokenizeString(s, ":")) + ss.push_back("@" + p); + builders = concatStringsSep(" ", ss); + } + +#if defined(__linux__) && defined(SANDBOX_SHELL) + sandboxPaths = tokenizeString("/bin/sh=" SANDBOX_SHELL); +#endif + + allowedImpureHostPrefixes = tokenizeString(DEFAULT_ALLOWED_IMPURE_PREFIXES); +} + +void loadConfFile() +{ + globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + + /* We only want to send overrides to the daemon, i.e. stuff from + ~/.nix/nix.conf or the command line. */ + globalConfig.resetOverriden(); + + auto dirs = getConfigDirs(); + // Iterate over them in reverse so that the ones appearing first in the path take priority + for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) { + globalConfig.applyConfigFile(*dir + "/nix/nix.conf"); + } +} + +unsigned int Settings::getDefaultCores() +{ + return std::max(1U, std::thread::hardware_concurrency()); +} + +StringSet Settings::getDefaultSystemFeatures() +{ + /* For backwards compatibility, accept some "features" that are + used in Nixpkgs to route builds to certain machines but don't + actually require anything special on the machines. */ + StringSet features{"nixos-test", "benchmark", "big-parallel"}; + + #if __linux__ + if (access("/dev/kvm", R_OK | W_OK) == 0) + features.insert("kvm"); + #endif + + return features; +} + +const string nixVersion = PACKAGE_VERSION; + +template<> void BaseSetting::set(const std::string & str) +{ + if (str == "true") value = smEnabled; + else if (str == "relaxed") value = smRelaxed; + else if (str == "false") value = smDisabled; + else throw UsageError("option '%s' has invalid value '%s'", name, str); +} + +template<> std::string BaseSetting::to_string() +{ + if (value == smEnabled) return "true"; + else if (value == smRelaxed) return "relaxed"; + else if (value == smDisabled) return "false"; + else abort(); +} + +template<> void BaseSetting::toJSON(JSONPlaceholder & out) +{ + AbstractSetting::toJSON(out); +} + +template<> void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.mkFlag() + .longName(name) + .description("Enable sandboxing.") + .handler([=](std::vector ss) { override(smEnabled); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description("Disable sandboxing.") + .handler([=](std::vector ss) { override(smDisabled); }) + .category(category); + args.mkFlag() + .longName("relaxed-" + name) + .description("Enable sandboxing, but allow builds to disable it.") + .handler([=](std::vector ss) { override(smRelaxed); }) + .category(category); +} + +void MaxBuildJobsSetting::set(const std::string & str) +{ + if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); + else if (!string2Int(str, value)) + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); +} + + +void initPlugins() +{ + for (const auto & pluginFile : settings.pluginFiles.get()) { + Paths pluginFiles; + try { + auto ents = readDirectory(pluginFile); + for (const auto & ent : ents) + pluginFiles.emplace_back(pluginFile + "/" + ent.name); + } catch (SysError & e) { + if (e.errNo != ENOTDIR) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ + void *handle = + dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + } + } + + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); +} + +} diff --git a/third_party/nix/src/libstore/globals.hh b/third_party/nix/src/libstore/globals.hh new file mode 100644 index 000000000..4ad9f6519 --- /dev/null +++ b/third_party/nix/src/libstore/globals.hh @@ -0,0 +1,370 @@ +#pragma once + +#include "types.hh" +#include "config.hh" +#include "util.hh" + +#include +#include + +#include + +namespace nix { + +typedef enum { smEnabled, smRelaxed, smDisabled } SandboxMode; + +struct MaxBuildJobsSetting : public BaseSetting +{ + MaxBuildJobsSetting(Config * options, + unsigned int def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void set(const std::string & str) override; +}; + +class Settings : public Config { + + unsigned int getDefaultCores(); + + StringSet getDefaultSystemFeatures(); + +public: + + Settings(); + + Path nixPrefix; + + /* The directory where we store sources and derived files. */ + Path nixStore; + + Path nixDataDir; /* !!! fix */ + + /* The directory where we log various operations. */ + Path nixLogDir; + + /* The directory where state is stored. */ + Path nixStateDir; + + /* The directory where configuration files are stored. */ + Path nixConfDir; + + /* The directory where internal helper programs are stored. */ + Path nixLibexecDir; + + /* The directory where the main programs are stored. */ + Path nixBinDir; + + /* The directory where the man pages are stored. */ + Path nixManDir; + + /* File name of the socket the daemon listens to. */ + Path nixDaemonSocketFile; + + Setting storeUri{this, getEnv("NIX_REMOTE", "auto"), "store", + "The default Nix store to use."}; + + Setting keepFailed{this, false, "keep-failed", + "Whether to keep temporary directories of failed builds."}; + + Setting keepGoing{this, false, "keep-going", + "Whether to keep building derivations when another build fails."}; + + Setting tryFallback{this, false, "fallback", + "Whether to fall back to building when substitution fails.", + {"build-fallback"}}; + + /* Whether to show build log output in real time. */ + bool verboseBuild = true; + + Setting logLines{this, 10, "log-lines", + "If verbose-build is false, the number of lines of the tail of " + "the log to show if a build fails."}; + + MaxBuildJobsSetting maxBuildJobs{this, 1, "max-jobs", + "Maximum number of parallel build jobs. \"auto\" means use number of cores.", + {"build-max-jobs"}}; + + Setting buildCores{this, getDefaultCores(), "cores", + "Number of CPU cores to utilize in parallel within a build, " + "i.e. by passing this number to Make via '-j'. 0 means that the " + "number of actual CPU cores on the local host ought to be " + "auto-detected.", {"build-cores"}}; + + /* Read-only mode. Don't copy stuff to the store, don't change + the database. */ + bool readOnlyMode = false; + + Setting thisSystem{this, SYSTEM, "system", + "The canonical Nix system name."}; + + Setting maxSilentTime{this, 0, "max-silent-time", + "The maximum time in seconds that a builer can go without " + "producing any output on stdout/stderr before it is killed. " + "0 means infinity.", + {"build-max-silent-time"}}; + + Setting buildTimeout{this, 0, "timeout", + "The maximum duration in seconds that a builder can run. " + "0 means infinity.", {"build-timeout"}}; + + PathSetting buildHook{this, true, nixLibexecDir + "/nix/build-remote", "build-hook", + "The path of the helper program that executes builds to remote machines."}; + + Setting builders{this, "@" + nixConfDir + "/machines", "builders", + "A semicolon-separated list of build machines, in the format of nix.machines."}; + + Setting buildersUseSubstitutes{this, false, "builders-use-substitutes", + "Whether build machines should use their own substitutes for obtaining " + "build dependencies if possible, rather than waiting for this host to " + "upload them."}; + + Setting reservedSize{this, 8 * 1024 * 1024, "gc-reserved-space", + "Amount of reserved disk space for the garbage collector."}; + + Setting fsyncMetadata{this, true, "fsync-metadata", + "Whether SQLite should use fsync()."}; + + Setting useSQLiteWAL{this, true, "use-sqlite-wal", + "Whether SQLite should use WAL mode."}; + + Setting syncBeforeRegistering{this, false, "sync-before-registering", + "Whether to call sync() before registering a path as valid."}; + + Setting useSubstitutes{this, true, "substitute", + "Whether to use substitutes.", + {"build-use-substitutes"}}; + + Setting buildUsersGroup{this, "", "build-users-group", + "The Unix group that contains the build users."}; + + Setting impersonateLinux26{this, false, "impersonate-linux-26", + "Whether to impersonate a Linux 2.6 machine on newer kernels.", + {"build-impersonate-linux-26"}}; + + Setting keepLog{this, true, "keep-build-log", + "Whether to store build logs.", + {"build-keep-log"}}; + + Setting compressLog{this, true, "compress-build-log", + "Whether to compress logs.", + {"build-compress-log"}}; + + Setting maxLogSize{this, 0, "max-build-log-size", + "Maximum number of bytes a builder can write to stdout/stderr " + "before being killed (0 means no limit).", + {"build-max-log-size"}}; + + /* When buildRepeat > 0 and verboseBuild == true, whether to print + repeated builds (i.e. builds other than the first one) to + stderr. Hack to prevent Hydra logs from being polluted. */ + bool printRepeatedBuilds = true; + + Setting pollInterval{this, 5, "build-poll-interval", + "How often (in seconds) to poll for locks."}; + + Setting checkRootReachability{this, false, "gc-check-reachability", + "Whether to check if new GC roots can in fact be found by the " + "garbage collector."}; + + Setting gcKeepOutputs{this, false, "keep-outputs", + "Whether the garbage collector should keep outputs of live derivations.", + {"gc-keep-outputs"}}; + + Setting gcKeepDerivations{this, true, "keep-derivations", + "Whether the garbage collector should keep derivers of live paths.", + {"gc-keep-derivations"}}; + + Setting autoOptimiseStore{this, false, "auto-optimise-store", + "Whether to automatically replace files with identical contents with hard links."}; + + Setting envKeepDerivations{this, false, "keep-env-derivations", + "Whether to add derivations as a dependency of user environments " + "(to prevent them from being GCed).", + {"env-keep-derivations"}}; + + /* Whether to lock the Nix client and worker to the same CPU. */ + bool lockCPU; + + /* Whether to show a stack trace if Nix evaluation fails. */ + Setting showTrace{this, false, "show-trace", + "Whether to show a stack trace on evaluation errors."}; + + Setting sandboxMode{this, + #if __linux__ + smEnabled + #else + smDisabled + #endif + , "sandbox", + "Whether to enable sandboxed builds. Can be \"true\", \"false\" or \"relaxed\".", + {"build-use-chroot", "build-use-sandbox"}}; + + Setting sandboxPaths{this, {}, "sandbox-paths", + "The paths to make available inside the build sandbox.", + {"build-chroot-dirs", "build-sandbox-paths"}}; + + Setting sandboxFallback{this, true, "sandbox-fallback", + "Whether to disable sandboxing when the kernel doesn't allow it."}; + + Setting extraSandboxPaths{this, {}, "extra-sandbox-paths", + "Additional paths to make available inside the build sandbox.", + {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}}; + + Setting buildRepeat{this, 0, "repeat", + "The number of times to repeat a build in order to verify determinism.", + {"build-repeat"}}; + +#if __linux__ + Setting sandboxShmSize{this, "50%", "sandbox-dev-shm-size", + "The size of /dev/shm in the build sandbox."}; + + Setting sandboxBuildDir{this, "/build", "sandbox-build-dir", + "The build directory inside the sandbox."}; +#endif + + Setting allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", + "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; + +#if __APPLE__ + Setting darwinLogSandboxViolations{this, false, "darwin-log-sandbox-violations", + "Whether to log Darwin sandbox access violations to the system log."}; +#endif + + Setting runDiffHook{this, false, "run-diff-hook", + "Whether to run the program specified by the diff-hook setting " + "repeated builds produce a different result. Typically used to " + "plug in diffoscope."}; + + PathSetting diffHook{this, true, "", "diff-hook", + "A program that prints out the differences between the two paths " + "specified on its command line."}; + + Setting enforceDeterminism{this, true, "enforce-determinism", + "Whether to fail if repeated builds produce different output."}; + + Setting trustedPublicKeys{this, + {"cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="}, + "trusted-public-keys", + "Trusted public keys for secure substitution.", + {"binary-cache-public-keys"}}; + + Setting secretKeyFiles{this, {}, "secret-key-files", + "Secret keys with which to sign local builds."}; + + Setting tarballTtl{this, 60 * 60, "tarball-ttl", + "How long downloaded files are considered up-to-date."}; + + Setting requireSigs{this, true, "require-sigs", + "Whether to check that any non-content-addressed path added to the " + "Nix store has a valid signature (that is, one signed using a key " + "listed in 'trusted-public-keys'."}; + + Setting extraPlatforms{this, + std::string{SYSTEM} == "x86_64-linux" ? StringSet{"i686-linux"} : StringSet{}, + "extra-platforms", + "Additional platforms that can be built on the local system. " + "These may be supported natively (e.g. armv7 on some aarch64 CPUs " + "or using hacks like qemu-user."}; + + Setting systemFeatures{this, getDefaultSystemFeatures(), + "system-features", + "Optional features that this system implements (like \"kvm\")."}; + + Setting substituters{this, + nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(), + "substituters", + "The URIs of substituters (such as https://cache.nixos.org/).", + {"binary-caches"}}; + + // FIXME: provide a way to add to option values. + Setting extraSubstituters{this, {}, "extra-substituters", + "Additional URIs of substituters.", + {"extra-binary-caches"}}; + + Setting trustedSubstituters{this, {}, "trusted-substituters", + "Disabled substituters that may be enabled via the substituters option by untrusted users.", + {"trusted-binary-caches"}}; + + Setting trustedUsers{this, {"root"}, "trusted-users", + "Which users or groups are trusted to ask the daemon to do unsafe things."}; + + Setting ttlNegativeNarInfoCache{this, 3600, "narinfo-cache-negative-ttl", + "The TTL in seconds for negative lookups in the disk cache i.e binary cache lookups that " + "return an invalid path result"}; + + Setting ttlPositiveNarInfoCache{this, 30 * 24 * 3600, "narinfo-cache-positive-ttl", + "The TTL in seconds for positive lookups in the disk cache i.e binary cache lookups that " + "return a valid path result."}; + + /* ?Who we trust to use the daemon in safe ways */ + Setting allowedUsers{this, {"*"}, "allowed-users", + "Which users or groups are allowed to connect to the daemon."}; + + Setting printMissing{this, true, "print-missing", + "Whether to print what paths need to be built or downloaded."}; + + Setting preBuildHook{this, +#if __APPLE__ + nixLibexecDir + "/nix/resolve-system-dependencies", +#else + "", +#endif + "pre-build-hook", + "A program to run just before a build to set derivation-specific build settings."}; + + Setting postBuildHook{this, "", "post-build-hook", + "A program to run just after each successful build."}; + + Setting netrcFile{this, fmt("%s/%s", nixConfDir, "netrc"), "netrc-file", + "Path to the netrc file used to obtain usernames/passwords for downloads."}; + + /* Path to the SSL CA file used */ + Path caFile; + +#if __linux__ + Setting filterSyscalls{this, true, "filter-syscalls", + "Whether to prevent certain dangerous system calls, such as " + "creation of setuid/setgid files or adding ACLs or extended " + "attributes. Only disable this if you're aware of the " + "security implications."}; + + Setting allowNewPrivileges{this, false, "allow-new-privileges", + "Whether builders can acquire new privileges by calling programs with " + "setuid/setgid bits or with file capabilities."}; +#endif + + Setting hashedMirrors{this, {"http://tarballs.nixos.org/"}, "hashed-mirrors", + "A list of servers used by builtins.fetchurl to fetch files by hash."}; + + Setting minFree{this, 0, "min-free", + "Automatically run the garbage collector when free disk space drops below the specified amount."}; + + Setting maxFree{this, std::numeric_limits::max(), "max-free", + "Stop deleting garbage when free disk space is above the specified amount."}; + + Setting minFreeCheckInterval{this, 5, "min-free-check-interval", + "Number of seconds between checking free disk space."}; + + Setting pluginFiles{this, {}, "plugin-files", + "Plugins to dynamically load at nix initialization time."}; +}; + + +// FIXME: don't use a global variable. +extern Settings settings; + +/* This should be called after settings are initialized, but before + anything else */ +void initPlugins(); + +void loadConfFile(); + +extern const string nixVersion; + +} diff --git a/third_party/nix/src/libstore/http-binary-cache-store.cc b/third_party/nix/src/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000..779f89e68 --- /dev/null +++ b/third_party/nix/src/libstore/http-binary-cache-store.cc @@ -0,0 +1,173 @@ +#include "binary-cache-store.hh" +#include "download.hh" +#include "globals.hh" +#include "nar-info-disk-cache.hh" + +namespace nix { + +MakeError(UploadToHTTP, Error); + +class HttpBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path cacheUri; + + struct State + { + bool enabled = true; + std::chrono::steady_clock::time_point disabledUntil; + }; + + Sync _state; + +public: + + HttpBinaryCacheStore( + const Params & params, const Path & _cacheUri) + : BinaryCacheStore(params) + , cacheUri(_cacheUri) + { + if (cacheUri.back() == '/') + cacheUri.pop_back(); + + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override + { + return cacheUri; + } + + void init() override + { + // FIXME: do this lazily? + if (!diskCache->cacheExists(cacheUri, wantMassQuery_, priority)) { + try { + BinaryCacheStore::init(); + } catch (UploadToHTTP &) { + throw Error("'%s' does not appear to be a binary cache", cacheUri); + } + diskCache->createCache(cacheUri, storeDir, wantMassQuery_, priority); + } + } + +protected: + + void maybeDisable() + { + auto state(_state.lock()); + if (state->enabled && settings.tryFallback) { + int t = 60; + printError("disabling binary cache '%s' for %s seconds", getUri(), t); + state->enabled = false; + state->disabledUntil = std::chrono::steady_clock::now() + std::chrono::seconds(t); + } + } + + void checkEnabled() + { + auto state(_state.lock()); + if (state->enabled) return; + if (std::chrono::steady_clock::now() > state->disabledUntil) { + state->enabled = true; + debug("re-enabling binary cache '%s'", getUri()); + return; + } + throw SubstituterDisabled("substituter '%s' is disabled", getUri()); + } + + bool fileExists(const std::string & path) override + { + checkEnabled(); + + try { + DownloadRequest request(cacheUri + "/" + path); + request.head = true; + getDownloader()->download(request); + return true; + } catch (DownloadError & e) { + /* S3 buckets return 403 if a file doesn't exist and the + bucket is unlistable, so treat 403 as 404. */ + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return false; + maybeDisable(); + throw; + } + } + + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override + { + auto req = DownloadRequest(cacheUri + "/" + path); + req.data = std::make_shared(data); // FIXME: inefficient + req.mimeType = mimeType; + try { + getDownloader()->download(req); + } catch (DownloadError & e) { + throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg()); + } + } + + DownloadRequest makeRequest(const std::string & path) + { + DownloadRequest request(cacheUri + "/" + path); + return request; + } + + void getFile(const std::string & path, Sink & sink) override + { + checkEnabled(); + auto request(makeRequest(path)); + try { + getDownloader()->download(std::move(request), sink); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + maybeDisable(); + throw; + } + } + + void getFile(const std::string & path, + Callback> callback) noexcept override + { + checkEnabled(); + + auto request(makeRequest(path)); + + auto callbackPtr = std::make_shared(std::move(callback)); + + getDownloader()->enqueueDownload(request, + {[callbackPtr, this](std::future result) { + try { + (*callbackPtr)(result.get().data); + } catch (DownloadError & e) { + if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden) + return (*callbackPtr)(std::shared_ptr()); + maybeDisable(); + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, 7) != "http://" && + std::string(uri, 0, 8) != "https://" && + (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") != "1" || std::string(uri, 0, 7) != "file://") + ) return 0; + auto store = std::make_shared(params, uri); + store->init(); + return store; +}); + +} + diff --git a/third_party/nix/src/libstore/legacy-ssh-store.cc b/third_party/nix/src/libstore/legacy-ssh-store.cc new file mode 100644 index 000000000..d5fbdd25a --- /dev/null +++ b/third_party/nix/src/libstore/legacy-ssh-store.cc @@ -0,0 +1,293 @@ +#include "archive.hh" +#include "pool.hh" +#include "remote-store.hh" +#include "serve-protocol.hh" +#include "store-api.hh" +#include "worker-protocol.hh" +#include "ssh.hh" +#include "derivations.hh" + +namespace nix { + +static std::string uriScheme = "ssh://"; + +struct LegacySSHStore : public Store +{ + const Setting maxConnections{this, 1, "max-connections", "maximum number of concurrent SSH connections"}; + const Setting sshKey{this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{this, false, "compress", "whether to compress the connection"}; + const Setting remoteProgram{this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"}; + const Setting remoteStore{this, "", "remote-store", "URI of the store on the remote system"}; + + // Hack for getting remote build log output. + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection + { + std::unique_ptr sshConn; + FdSink to; + FdSource from; + int remoteVersion; + bool good = true; + }; + + std::string host; + + ref> connections; + + SSHMaster master; + + LegacySSHStore(const string & host, const Params & params) + : Store(params) + , host(host) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnection(); }, + [](const ref & r) { return r->good; } + )) + , master( + host, + sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress, + logFD) + { + } + + ref openConnection() + { + auto conn = make_ref(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + unsigned int magic = readInt(conn->from); + if (magic != SERVE_MAGIC_2) + throw Error("protocol mismatch with 'nix-store --serve' on '%s'", host); + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + + } catch (EndOfFile & e) { + throw Error("cannot connect to '%1%'", host); + } + + return conn; + }; + + string getUri() override + { + return uriScheme + host; + } + + void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override + { + try { + auto conn(connections->get()); + + debug("querying remote host '%s' for info on '%s'", host, path); + + conn->to << cmdQueryPathInfos << PathSet{path}; + conn->to.flush(); + + auto info = std::make_shared(); + conn->from >> info->path; + if (info->path.empty()) return callback(nullptr); + assert(path == info->path); + + PathSet references; + conn->from >> info->deriver; + info->references = readStorePaths(*this, conn->from); + readLongLong(conn->from); // download size + info->narSize = readLongLong(conn->from); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) { + auto s = readString(conn->from); + info->narHash = s.empty() ? Hash() : Hash(s); + conn->from >> info->ca; + info->sigs = readStrings(conn->from); + } + + auto s = readString(conn->from); + assert(s == ""); + + callback(std::move(info)); + } catch (...) { callback.rethrow(); } + } + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) override + { + debug("adding path '%s' to remote host '%s'", info.path, host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << cmdAddToStoreNar + << info.path + << info.deriver + << info.narHash.to_string(Base16, false) + << info.references + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << info.ca; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + + conn->to + << cmdImportPaths + << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to + << exportMagic + << info.path + << info.references + << info.deriver + << 0 + << 0; + conn->to.flush(); + + } + + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s', info.path, host"); + } + + void narFromPath(const Path & path, Sink & sink) override + { + auto conn(connections->get()); + + conn->to << cmdDumpStorePath << path; + conn->to.flush(); + copyNAR(conn->from, sink); + } + + Path queryPathFromHashPart(const string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + Path addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override + { unsupported("addToStore"); } + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) override + { unsupported("addTextToStore"); } + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override + { + auto conn(connections->get()); + + conn->to + << cmdBuildDerivation + << drvPath + << drv + << settings.maxSilentTime + << settings.buildTimeout; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 2) + conn->to + << settings.maxLogSize; + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->to + << settings.buildRepeat + << settings.enforceDeterminism; + + conn->to.flush(); + + BuildResult status; + status.status = (BuildResult::Status) readInt(conn->from); + conn->from >> status.errorMsg; + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3) + conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + + return status; + } + + void ensurePath(const Path & path) override + { unsupported("ensurePath"); } + + void computeFSClosure(const PathSet & paths, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override + { + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to + << cmdQueryClosure + << includeOutputs + << paths; + conn->to.flush(); + + auto res = readStorePaths(*this, conn->from); + + out.insert(res.begin(), res.end()); + } + + PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override + { + auto conn(connections->get()); + + conn->to + << cmdQueryValidPaths + << false // lock + << maybeSubstitute + << paths; + conn->to.flush(); + + return readStorePaths(*this, conn->from); + } + + void connect() override + { + auto conn(connections->get()); + } + + unsigned int getProtocol() override + { + auto conn(connections->get()); + return conn->remoteVersion; + } +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared(std::string(uri, uriScheme.size()), params); +}); + +} diff --git a/third_party/nix/src/libstore/local-binary-cache-store.cc b/third_party/nix/src/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000..b7001795b --- /dev/null +++ b/third_party/nix/src/libstore/local-binary-cache-store.cc @@ -0,0 +1,103 @@ +#include "binary-cache-store.hh" +#include "globals.hh" +#include "nar-info-disk-cache.hh" + +namespace nix { + +class LocalBinaryCacheStore : public BinaryCacheStore +{ +private: + + Path binaryCacheDir; + +public: + + LocalBinaryCacheStore( + const Params & params, const Path & binaryCacheDir) + : BinaryCacheStore(params) + , binaryCacheDir(binaryCacheDir) + { + } + + void init() override; + + std::string getUri() override + { + return "file://" + binaryCacheDir; + } + +protected: + + bool fileExists(const std::string & path) override; + + void upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) override; + + void getFile(const std::string & path, Sink & sink) override + { + try { + readFile(binaryCacheDir + "/" + path, sink); + } catch (SysError & e) { + if (e.errNo == ENOENT) + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache", path); + } + } + + PathSet queryAllValidPaths() override + { + PathSet paths; + + for (auto & entry : readDirectory(binaryCacheDir)) { + if (entry.name.size() != 40 || + !hasSuffix(entry.name, ".narinfo")) + continue; + paths.insert(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)); + } + + return paths; + } + +}; + +void LocalBinaryCacheStore::init() +{ + createDirs(binaryCacheDir + "/nar"); + BinaryCacheStore::init(); +} + +static void atomicWrite(const Path & path, const std::string & s) +{ + Path tmp = path + ".tmp." + std::to_string(getpid()); + AutoDelete del(tmp, false); + writeFile(tmp, s); + if (rename(tmp.c_str(), path.c_str())) + throw SysError(format("renaming '%1%' to '%2%'") % tmp % path); + del.cancel(); +} + +bool LocalBinaryCacheStore::fileExists(const std::string & path) +{ + return pathExists(binaryCacheDir + "/" + path); +} + +void LocalBinaryCacheStore::upsertFile(const std::string & path, + const std::string & data, + const std::string & mimeType) +{ + atomicWrite(binaryCacheDir + "/" + path, data); +} + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (getEnv("_NIX_FORCE_HTTP_BINARY_CACHE_STORE") == "1" || + std::string(uri, 0, 7) != "file://") + return 0; + auto store = std::make_shared(params, std::string(uri, 7)); + store->init(); + return store; +}); + +} diff --git a/third_party/nix/src/libstore/local-fs-store.cc b/third_party/nix/src/libstore/local-fs-store.cc new file mode 100644 index 000000000..642e4070d --- /dev/null +++ b/third_party/nix/src/libstore/local-fs-store.cc @@ -0,0 +1,131 @@ +#include "archive.hh" +#include "fs-accessor.hh" +#include "store-api.hh" +#include "globals.hh" +#include "compression.hh" +#include "derivations.hh" + +namespace nix { + +LocalFSStore::LocalFSStore(const Params & params) + : Store(params) +{ +} + +struct LocalStoreAccessor : public FSAccessor +{ + ref store; + + LocalStoreAccessor(ref store) : store(store) { } + + Path toRealPath(const Path & path) + { + Path storePath = store->toStorePath(path); + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + } + + FSAccessor::Stat stat(const Path & path) override + { + auto realPath = toRealPath(path); + + struct stat st; + if (lstat(realPath.c_str(), &st)) { + if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; + throw SysError(format("getting status of '%1%'") % path); + } + + if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) + throw Error(format("file '%1%' has unsupported type") % path); + + return { + S_ISREG(st.st_mode) ? Type::tRegular : + S_ISLNK(st.st_mode) ? Type::tSymlink : + Type::tDirectory, + S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, + S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + } + + StringSet readDirectory(const Path & path) override + { + auto realPath = toRealPath(path); + + auto entries = nix::readDirectory(realPath); + + StringSet res; + for (auto & entry : entries) + res.insert(entry.name); + + return res; + } + + std::string readFile(const Path & path) override + { + return nix::readFile(toRealPath(path)); + } + + std::string readLink(const Path & path) override + { + return nix::readLink(toRealPath(path)); + } +}; + +ref LocalFSStore::getFSAccessor() +{ + return make_ref(ref( + std::dynamic_pointer_cast(shared_from_this()))); +} + +void LocalFSStore::narFromPath(const Path & path, Sink & sink) +{ + if (!isValidPath(path)) + throw Error(format("path '%s' is not valid") % path); + dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink); +} + +const string LocalFSStore::drvsLogDir = "drvs"; + + + +std::shared_ptr LocalFSStore::getBuildLog(const Path & path_) +{ + auto path(path_); + + assertStorePath(path); + + + if (!isDerivation(path)) { + try { + path = queryPathInfo(path)->deriver; + } catch (InvalidPath &) { + return nullptr; + } + if (path == "") return nullptr; + } + + string baseName = baseNameOf(path); + + for (int j = 0; j < 2; j++) { + + Path logPath = + j == 0 + ? fmt("%s/%s/%s/%s", logDir, drvsLogDir, string(baseName, 0, 2), string(baseName, 2)) + : fmt("%s/%s/%s", logDir, drvsLogDir, baseName); + Path logBz2Path = logPath + ".bz2"; + + if (pathExists(logPath)) + return std::make_shared(readFile(logPath)); + + else if (pathExists(logBz2Path)) { + try { + return decompress("bzip2", readFile(logBz2Path)); + } catch (Error &) { } + } + + } + + return nullptr; +} + +} diff --git a/third_party/nix/src/libstore/local-store.cc b/third_party/nix/src/libstore/local-store.cc new file mode 100644 index 000000000..84ddd964b --- /dev/null +++ b/third_party/nix/src/libstore/local-store.cc @@ -0,0 +1,1453 @@ +#include "local-store.hh" +#include "globals.hh" +#include "archive.hh" +#include "pathlocks.hh" +#include "worker-protocol.hh" +#include "derivations.hh" +#include "nar-info.hh" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __linux__ +#include +#include +#include +#include +#include +#endif + +#ifdef __CYGWIN__ +#include +#endif + +#include + + +namespace nix { + + +LocalStore::LocalStore(const Params & params) + : Store(params) + , LocalFSStore(params) + , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", + "physical path to the Nix store"} + , realStoreDir(realStoreDir_) + , dbDir(stateDir + "/db") + , linksDir(realStoreDir + "/.links") + , reservedPath(dbDir + "/reserved") + , schemaPath(dbDir + "/schema") + , trashDir(realStoreDir + "/trash") + , tempRootsDir(stateDir + "/temproots") + , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) +{ + auto state(_state.lock()); + + /* Create missing state directories if they don't already exist. */ + createDirs(realStoreDir); + makeStoreWritable(); + createDirs(linksDir); + Path profilesDir = stateDir + "/profiles"; + createDirs(profilesDir); + createDirs(tempRootsDir); + createDirs(dbDir); + Path gcRootsDir = stateDir + "/gcroots"; + if (!pathExists(gcRootsDir)) { + createDirs(gcRootsDir); + createSymlink(profilesDir, gcRootsDir + "/profiles"); + } + + for (auto & perUserDir : {profilesDir + "/per-user", gcRootsDir + "/per-user"}) { + createDirs(perUserDir); + if (chmod(perUserDir.c_str(), 0755) == -1) + throw SysError("could not set permissions on '%s' to 755", perUserDir); + } + + createUser(getUserName(), getuid()); + + /* Optionally, create directories and set permissions for a + multi-user install. */ + if (getuid() == 0 && settings.buildUsersGroup != "") { + mode_t perm = 01775; + + struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); + if (!gr) + printError(format("warning: the group '%1%' specified in 'build-users-group' does not exist") + % settings.buildUsersGroup); + else { + struct stat st; + if (stat(realStoreDir.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % realStoreDir); + + if (st.st_uid != 0 || st.st_gid != gr->gr_gid || (st.st_mode & ~S_IFMT) != perm) { + if (chown(realStoreDir.c_str(), 0, gr->gr_gid) == -1) + throw SysError(format("changing ownership of path '%1%'") % realStoreDir); + if (chmod(realStoreDir.c_str(), perm) == -1) + throw SysError(format("changing permissions on path '%1%'") % realStoreDir); + } + } + } + + /* Ensure that the store and its parents are not symlinks. */ + if (getEnv("NIX_IGNORE_SYMLINK_STORE") != "1") { + Path path = realStoreDir; + struct stat st; + while (path != "/") { + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of '%1%'") % path); + if (S_ISLNK(st.st_mode)) + throw Error(format( + "the path '%1%' is a symlink; " + "this is not allowed for the Nix store and its parent directories") + % path); + path = dirOf(path); + } + } + + /* We can't open a SQLite database if the disk is full. Since + this prevents the garbage collector from running when it's most + needed, we reserve some dummy space that we can free just + before doing a garbage collection. */ + try { + struct stat st; + if (stat(reservedPath.c_str(), &st) == -1 || + st.st_size != settings.reservedSize) + { + AutoCloseFD fd = open(reservedPath.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0600); + int res = -1; +#if HAVE_POSIX_FALLOCATE + res = posix_fallocate(fd.get(), 0, settings.reservedSize); +#endif + if (res == -1) { + writeFull(fd.get(), string(settings.reservedSize, 'X')); + [[gnu::unused]] auto res2 = ftruncate(fd.get(), settings.reservedSize); + } + } + } catch (SysError & e) { /* don't care about errors */ + } + + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ + Path globalLockPath = dbDir + "/big-lock"; + globalLock = openLockFile(globalLockPath.c_str(), true); + + if (!lockFile(globalLock.get(), ltRead, false)) { + printError("waiting for the big Nix store lock..."); + lockFile(globalLock.get(), ltRead, true); + } + + /* Check the current database schema and if necessary do an + upgrade. */ + int curSchema = getSchema(); + if (curSchema > nixSchemaVersion) + throw Error(format("current Nix store schema is version %1%, but I only support %2%") + % curSchema % nixSchemaVersion); + + else if (curSchema == 0) { /* new store */ + curSchema = nixSchemaVersion; + openDB(*state, true); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + } + + else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + + if (curSchema < 6) + throw Error( + "Your Nix store has a database in flat file format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 1.11 first."); + + if (!lockFile(globalLock.get(), ltWrite, false)) { + printError("waiting for exclusive access to the Nix store..."); + lockFile(globalLock.get(), ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 7) { upgradeStore7(); } + + openDB(*state, false); + + if (curSchema < 8) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ultimate integer"); + state->db.exec("alter table ValidPaths add column sigs text"); + txn.commit(); + } + + if (curSchema < 9) { + SQLiteTxn txn(state->db); + state->db.exec("drop table FailedPaths"); + txn.commit(); + } + + if (curSchema < 10) { + SQLiteTxn txn(state->db); + state->db.exec("alter table ValidPaths add column ca text"); + txn.commit(); + } + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock.get(), ltRead, true); + } + + else openDB(*state, false); + + /* Prepare SQL statements. */ + state->stmtRegisterValidPath.create(state->db, + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);"); + state->stmtUpdatePathInfo.create(state->db, + "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;"); + state->stmtAddReference.create(state->db, + "insert or replace into Refs (referrer, reference) values (?, ?);"); + state->stmtQueryPathInfo.create(state->db, + "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;"); + state->stmtQueryReferences.create(state->db, + "select path from Refs join ValidPaths on reference = id where referrer = ?;"); + state->stmtQueryReferrers.create(state->db, + "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); + state->stmtInvalidatePath.create(state->db, + "delete from ValidPaths where path = ?;"); + state->stmtAddDerivationOutput.create(state->db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); + state->stmtQueryValidDerivers.create(state->db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); + state->stmtQueryDerivationOutputs.create(state->db, + "select id, path from DerivationOutputs where drv = ?;"); + // Use "path >= ?" with limit 1 rather than "path like '?%'" to + // ensure efficient lookup. + state->stmtQueryPathFromHashPart.create(state->db, + "select path from ValidPaths where path >= ? limit 1;"); + state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths"); +} + + +LocalStore::~LocalStore() +{ + std::shared_future future; + + { + auto state(_state.lock()); + if (state->gcRunning) + future = state->gcFuture; + } + + if (future.valid()) { + printError("waiting for auto-GC to finish on exit..."); + future.get(); + } + + try { + auto state(_state.lock()); + if (state->fdTempRoots) { + state->fdTempRoots = -1; + unlink(fnTempRoots.c_str()); + } + } catch (...) { + ignoreException(); + } +} + + +std::string LocalStore::getUri() +{ + return "local"; +} + + +int LocalStore::getSchema() +{ + int curSchema = 0; + if (pathExists(schemaPath)) { + string s = readFile(schemaPath); + if (!string2Int(s, curSchema)) + throw Error(format("'%1%' is corrupt") % schemaPath); + } + return curSchema; +} + + +void LocalStore::openDB(State & state, bool create) +{ + if (access(dbDir.c_str(), R_OK | W_OK)) + throw SysError(format("Nix database directory '%1%' is not writable") % dbDir); + + /* Open the Nix database. */ + string dbPath = dbDir + "/db.sqlite"; + auto & db(state.db); + if (sqlite3_open_v2(dbPath.c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error(format("cannot open Nix database '%1%'") % dbPath); + +#ifdef __CYGWIN__ + /* The cygwin version of sqlite3 has a patch which calls + SetDllDirectory("/usr/bin") on init. It was intended to fix extension + loading, which we don't use, and the effect of SetDllDirectory is + inherited by child processes, and causes libraries to be loaded from + /usr/bin instead of $PATH. This breaks quite a few things (e.g. + checkPhase on openssh), so we set it back to default behaviour. */ + SetDllDirectoryW(L""); +#endif + + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(db, "setting timeout"); + + db.exec("pragma foreign_keys = 1"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = settings.fsyncMetadata ? "normal" : "off"; + db.exec("pragma synchronous = " + syncMode); + + /* Set the SQLite journal mode. WAL mode is fastest, so it's the + default. */ + string mode = settings.useSQLiteWAL ? "wal" : "truncate"; + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) + throwSQLiteError(db, "querying journal mode"); + prevMode = string((const char *) sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting journal mode"); + + /* Increase the auto-checkpoint interval to 40000 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 40000;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "setting autocheckpoint interval"); + + /* Initialise the database schema, if necessary. */ + if (create) { + const char * schema = +#include "schema.sql.gen.hh" + ; + db.exec(schema); + } +} + + +/* To improve purity, users may want to make the Nix store a read-only + bind mount. So make the Nix store writable for this process. */ +void LocalStore::makeStoreWritable() +{ +#if __linux__ + if (getuid() != 0) return; + /* Check if /nix/store is on a read-only mount. */ + struct statvfs stat; + if (statvfs(realStoreDir.c_str(), &stat) != 0) + throw SysError("getting info about the Nix store mount point"); + + if (stat.f_flag & ST_RDONLY) { + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + if (mount(0, realStoreDir.c_str(), "none", MS_REMOUNT | MS_BIND, 0) == -1) + throw SysError(format("remounting %1% writable") % realStoreDir); + } +#endif +} + + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError(format("changing mode of '%1%' to %2$o") % path % mode); + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError(format("changing modification time of '%1%'") % path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + canonicaliseTimestampAndPermissions(path, st); +} + + +static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +{ + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError(format("clearing flags of path '%1%'") % path); + } +#endif + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error(format("file '%1%' has an unsupported type") % path); + +#if __linux__ + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + /* Ignore SELinux security labels since these cannot be + removed even by root. */ + if (eaName == "security.selinux") continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (fromUid != (uid_t) -1 && st.st_uid != fromUid) { + assert(!S_ISDIR(st.st_mode)); + if (inodesSeen.find(Inode(st.st_dev, st.st_ino)) == inodesSeen.end()) + throw BuildError(format("invalid ownership on file '%1%'") % path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError(format("changing owner of '%1%' to %2%") + % path % geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto & i : entries) + canonicalisePathMetaData_(path + "/" + i.name, fromUid, inodesSeen); + } +} + + +void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, fromUid, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error(format("wrong ownership of top-level store path '%1%'") % path); + } +} + + +void canonicalisePathMetaData(const Path & path, uid_t fromUid) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, fromUid, inodesSeen); +} + + +void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation & drv) +{ + string drvName = storePathToName(drvPath); + assert(isDerivation(drvName)); + drvName = string(drvName, 0, drvName.size() - drvExtension.size()); + + if (drv.isFixedOutput()) { + DerivationOutputs::const_iterator out = drv.outputs.find("out"); + if (out == drv.outputs.end()) + throw Error(format("derivation '%1%' does not have an output named 'out'") % drvPath); + + bool recursive; Hash h; + out->second.parseHashInfo(recursive, h); + Path outPath = makeFixedOutputPath(recursive, h, drvName); + + StringPairs::const_iterator j = drv.env.find("out"); + if (out->second.path != outPath || j == drv.env.end() || j->second != outPath) + throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") + % drvPath % out->second.path % outPath); + } + + else { + Derivation drvCopy(drv); + for (auto & i : drvCopy.outputs) { + i.second.path = ""; + drvCopy.env[i.first] = ""; + } + + Hash h = hashDerivationModulo(*this, drvCopy); + + for (auto & i : drv.outputs) { + Path outPath = makeOutputPath(i.first, h, drvName); + StringPairs::const_iterator j = drv.env.find(i.first); + if (i.second.path != outPath || j == drv.env.end() || j->second != outPath) + throw Error(format("derivation '%1%' has incorrect output '%2%', should be '%3%'") + % drvPath % i.second.path % outPath); + } + } +} + + +uint64_t LocalStore::addValidPath(State & state, + const ValidPathInfo & info, bool checkOutputs) +{ + if (info.ca != "" && !info.isContentAddressed(*this)) + throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't", info.path); + + state.stmtRegisterValidPath.use() + (info.path) + (info.narHash.to_string(Base16)) + (info.registrationTime == 0 ? time(0) : info.registrationTime) + (info.deriver, info.deriver != "") + (info.narSize, info.narSize != 0) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.ca, !info.ca.empty()) + .exec(); + uint64_t id = sqlite3_last_insert_rowid(state.db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(info.path)); + + /* Verify that the output paths in the derivation are correct + (i.e., follow the scheme for computing output paths from + derivations). Note that if this throws an error, then the + DB transaction is rolled back, so the path validity + registration above is undone. */ + if (checkOutputs) checkDerivationOutputs(info.path, drv); + + for (auto & i : drv.outputs) { + state.stmtAddDerivationOutput.use() + (id) + (i.first) + (i.second.path) + .exec(); + } + } + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.upsert(storePathToHash(info.path), std::make_shared(info)); + } + + return id; +} + + +void LocalStore::queryPathInfoUncached(const Path & path, + Callback> callback) noexcept +{ + try { + auto info = std::make_shared(); + info->path = path; + + assertStorePath(path); + + callback(retrySQLite>([&]() { + auto state(_state.lock()); + + /* Get the path info. */ + auto useQueryPathInfo(state->stmtQueryPathInfo.use()(path)); + + if (!useQueryPathInfo.next()) + return std::shared_ptr(); + + info->id = useQueryPathInfo.getInt(0); + + try { + info->narHash = Hash(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("in valid-path entry for '%s': %s", path, e.what()); + } + + info->registrationTime = useQueryPathInfo.getInt(2); + + auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3); + if (s) info->deriver = s; + + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); + + info->ultimate = useQueryPathInfo.getInt(5) == 1; + + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6); + if (s) info->sigs = tokenizeString(s, " "); + + s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7); + if (s) info->ca = s; + + /* Get the references. */ + auto useQueryReferences(state->stmtQueryReferences.use()(info->id)); + + while (useQueryReferences.next()) + info->references.insert(useQueryReferences.getStr(0)); + + return info; + })); + + } catch (...) { callback.rethrow(); } +} + + +/* Update path info in the database. */ +void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info) +{ + state.stmtUpdatePathInfo.use() + (info.narSize, info.narSize != 0) + (info.narHash.to_string(Base16)) + (info.ultimate ? 1 : 0, info.ultimate) + (concatStringsSep(" ", info.sigs), !info.sigs.empty()) + (info.ca, !info.ca.empty()) + (info.path) + .exec(); +} + + +uint64_t LocalStore::queryValidPathId(State & state, const Path & path) +{ + auto use(state.stmtQueryPathInfo.use()(path)); + if (!use.next()) + throw Error(format("path '%1%' is not valid") % path); + return use.getInt(0); +} + + +bool LocalStore::isValidPath_(State & state, const Path & path) +{ + return state.stmtQueryPathInfo.use()(path).next(); +} + + +bool LocalStore::isValidPathUncached(const Path & path) +{ + return retrySQLite([&]() { + auto state(_state.lock()); + return isValidPath_(*state, path); + }); +} + + +PathSet LocalStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +{ + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; +} + + +PathSet LocalStore::queryAllValidPaths() +{ + return retrySQLite([&]() { + auto state(_state.lock()); + auto use(state->stmtQueryValidPaths.use()); + PathSet res; + while (use.next()) res.insert(use.getStr(0)); + return res; + }); +} + + +void LocalStore::queryReferrers(State & state, const Path & path, PathSet & referrers) +{ + auto useQueryReferrers(state.stmtQueryReferrers.use()(path)); + + while (useQueryReferrers.next()) + referrers.insert(useQueryReferrers.getStr(0)); +} + + +void LocalStore::queryReferrers(const Path & path, PathSet & referrers) +{ + assertStorePath(path); + return retrySQLite([&]() { + auto state(_state.lock()); + queryReferrers(*state, path, referrers); + }); +} + + +PathSet LocalStore::queryValidDerivers(const Path & path) +{ + assertStorePath(path); + + return retrySQLite([&]() { + auto state(_state.lock()); + + auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(path)); + + PathSet derivers; + while (useQueryValidDerivers.next()) + derivers.insert(useQueryValidDerivers.getStr(1)); + + return derivers; + }); +} + + +PathSet LocalStore::queryDerivationOutputs(const Path & path) +{ + return retrySQLite([&]() { + auto state(_state.lock()); + + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); + + PathSet outputs; + while (useQueryDerivationOutputs.next()) + outputs.insert(useQueryDerivationOutputs.getStr(1)); + + return outputs; + }); +} + + +StringSet LocalStore::queryDerivationOutputNames(const Path & path) +{ + return retrySQLite([&]() { + auto state(_state.lock()); + + auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use() + (queryValidPathId(*state, path))); + + StringSet outputNames; + while (useQueryDerivationOutputs.next()) + outputNames.insert(useQueryDerivationOutputs.getStr(0)); + + return outputNames; + }); +} + + +Path LocalStore::queryPathFromHashPart(const string & hashPart) +{ + if (hashPart.size() != storePathHashLen) throw Error("invalid hash part"); + + Path prefix = storeDir + "/" + hashPart; + + return retrySQLite([&]() -> std::string { + auto state(_state.lock()); + + auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix)); + + if (!useQueryPathFromHashPart.next()) return ""; + + const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0); + return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : ""; + }); +} + + +PathSet LocalStore::querySubstitutablePaths(const PathSet & paths) +{ + if (!settings.useSubstitutes) return PathSet(); + + auto remaining = paths; + PathSet res; + + for (auto & sub : getDefaultSubstituters()) { + if (remaining.empty()) break; + if (sub->storeDir != storeDir) continue; + if (!sub->wantMassQuery()) continue; + + auto valid = sub->queryValidPaths(remaining); + + PathSet remaining2; + for (auto & path : remaining) + if (valid.count(path)) + res.insert(path); + else + remaining2.insert(path); + + std::swap(remaining, remaining2); + } + + return res; +} + + +void LocalStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + if (!settings.useSubstitutes) return; + for (auto & sub : getDefaultSubstituters()) { + if (sub->storeDir != storeDir) continue; + for (auto & path : paths) { + if (infos.count(path)) continue; + debug(format("checking substituter '%s' for path '%s'") + % sub->getUri() % path); + try { + auto info = sub->queryPathInfo(path); + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + infos[path] = SubstitutablePathInfo{ + info->deriver, + info->references, + narInfo ? narInfo->fileSize : 0, + info->narSize}; + } catch (InvalidPath &) { + } catch (SubstituterDisabled &) { + } catch (Error & e) { + if (settings.tryFallback) + printError(e.what()); + else + throw; + } + } + } +} + + +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + + +void LocalStore::registerValidPaths(const ValidPathInfos & infos) +{ + /* SQLite will fsync by default, but the new valid paths may not + be fsync-ed. So some may want to fsync them before registering + the validity, at the expense of some speed of the path + registering operation. */ + if (settings.syncBeforeRegistering) sync(); + + return retrySQLite([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + PathSet paths; + + for (auto & i : infos) { + assert(i.narHash.type == htSHA256); + if (isValidPath_(*state, i.path)) + updatePathInfo(*state, i); + else + addValidPath(*state, i, false); + paths.insert(i.path); + } + + for (auto & i : infos) { + auto referrer = queryValidPathId(*state, i.path); + for (auto & j : i.references) + state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec(); + } + + /* Check that the derivation outputs are correct. We can't do + this in addValidPath() above, because the references might + not be valid yet. */ + for (auto & i : infos) + if (isDerivation(i.path)) { + // FIXME: inefficient; we already loaded the + // derivation in addValidPath(). + Derivation drv = readDerivation(realStoreDir + "/" + baseNameOf(i.path)); + checkDerivationOutputs(i.path, drv); + } + + /* Do a topological sort of the paths. This will throw an + error if a cycle is detected and roll back the + transaction. Cycles can only occur when a derivation + has multiple outputs. */ + topoSortPaths(paths); + + txn.commit(); + }); +} + + +/* Invalidate a path. The caller is responsible for checking that + there are no referrers. */ +void LocalStore::invalidatePath(State & state, const Path & path) +{ + debug(format("invalidating path '%1%'") % path); + + state.stmtInvalidatePath.use()(path).exec(); + + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.erase(storePathToHash(path)); + } +} + + +const PublicKeys & LocalStore::getPublicKeys() +{ + auto state(_state.lock()); + if (!state->publicKeys) + state->publicKeys = std::make_unique(getDefaultPublicKeys()); + return *state->publicKeys; +} + + +void LocalStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) +{ + if (!info.narHash) + throw Error("cannot add path '%s' because it lacks a hash", info.path); + + if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys())) + throw Error("cannot add path '%s' because it lacks a valid signature", info.path); + + addTempRoot(info.path); + + if (repair || !isValidPath(info.path)) { + + PathLocks outputLock; + + Path realPath = realStoreDir + "/" + baseNameOf(info.path); + + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + if (!locksHeld.count(info.path)) + outputLock.lockPaths({realPath}); + + if (repair || !isValidPath(info.path)) { + + deletePath(realPath); + + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(htSHA256); + + LambdaSource wrapperSource([&](unsigned char * data, size_t len) -> size_t { + size_t n = source.read(data, len); + hashSink(data, n); + return n; + }); + + restorePath(realPath, wrapperSource); + + auto hashResult = hashSink.finish(); + + if (hashResult.first != info.narHash) + throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narHash.to_string(), hashResult.first.to_string()); + + if (hashResult.second != info.narSize) + throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s", + info.path, info.narSize, hashResult.second); + + autoGC(); + + canonicalisePathMetaData(realPath, -1); + + optimisePath(realPath); // FIXME: combine with hashPath() + + registerValidPath(info); + } + + outputLock.setDeletion(true); + } +} + + +Path LocalStore::addToStoreFromDump(const string & dump, const string & name, + bool recursive, HashType hashAlgo, RepairFlag repair) +{ + Hash h = hashString(hashAlgo, dump); + + Path dstPath = makeFixedOutputPath(recursive, h, name); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + /* The first check above is an optimisation to prevent + unnecessary lock acquisition. */ + + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if (repair || !isValidPath(dstPath)) { + + deletePath(realPath); + + autoGC(); + + if (recursive) { + StringSource source(dump); + restorePath(realPath, source); + } else + writeFile(realPath, dump); + + canonicalisePathMetaData(realPath, -1); + + /* Register the SHA-256 hash of the NAR serialisation of + the path in the database. We may just have computed it + above (if called with recursive == true and hashAlgo == + sha256); otherwise, compute it here. */ + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, realPath); + + optimisePath(realPath); // FIXME: combine with hashPath() + + ValidPathInfo info; + info.path = dstPath; + info.narHash = hash.first; + info.narSize = hash.second; + info.ca = makeFixedOutputCA(recursive, h); + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +Path LocalStore::addToStore(const string & name, const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + Path srcPath(absPath(_srcPath)); + + /* Read the whole path into memory. This is not a very scalable + method for very large paths, but `copyPath' is mainly used for + small files. */ + StringSink sink; + if (recursive) + dumpPath(srcPath, sink, filter); + else + sink.s = make_ref(readFile(srcPath)); + + return addToStoreFromDump(*sink.s, name, recursive, hashAlgo, repair); +} + + +Path LocalStore::addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) +{ + auto hash = hashString(htSHA256, s); + auto dstPath = makeTextPath(name, hash, references); + + addTempRoot(dstPath); + + if (repair || !isValidPath(dstPath)) { + + Path realPath = realStoreDir + "/" + baseNameOf(dstPath); + + PathLocks outputLock({realPath}); + + if (repair || !isValidPath(dstPath)) { + + deletePath(realPath); + + autoGC(); + + writeFile(realPath, s); + + canonicalisePathMetaData(realPath, -1); + + StringSink sink; + dumpString(s, sink); + auto narHash = hashString(htSHA256, *sink.s); + + optimisePath(realPath); + + ValidPathInfo info; + info.path = dstPath; + info.narHash = narHash; + info.narSize = sink.s->size(); + info.references = references; + info.ca = "text:" + hash.to_string(); + registerValidPath(info); + } + + outputLock.setDeletion(true); + } + + return dstPath; +} + + +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() +{ + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(realStoreDir); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + + +void LocalStore::invalidatePathChecked(const Path & path) +{ + assertStorePath(path); + + retrySQLite([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + if (isValidPath_(*state, path)) { + PathSet referrers; queryReferrers(*state, path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse(format("cannot delete path '%1%' because it is in use by %2%") + % path % showPaths(referrers)); + invalidatePath(*state, path); + } + + txn.commit(); + }); +} + + +bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) +{ + printError(format("reading the Nix store...")); + + bool errors = false; + + /* Acquire the global GC lock to get a consistent snapshot of + existing and valid paths. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); + + PathSet store; + for (auto & i : readDirectory(realStoreDir)) store.insert(i.name); + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + PathSet validPaths2 = queryAllValidPaths(), validPaths, done; + + fdGCLock = -1; + + for (auto & i : validPaths2) + verifyPath(i, store, done, validPaths, repair, errors); + + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + printInfo("checking hashes..."); + + Hash nullHash(htSHA256); + + for (auto & i : validPaths) { + try { + auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(i))); + + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of '%1%'") % i); + HashResult current = hashPath(info->narHash.type, toRealPath(i)); + + if (info->narHash != nullHash && info->narHash != current.first) { + printError(format("path '%1%' was modified! " + "expected hash '%2%', got '%3%'") + % i % info->narHash.to_string() % current.first.to_string()); + if (repair) repairPath(i); else errors = true; + } else { + + bool update = false; + + /* Fill in missing hashes. */ + if (info->narHash == nullHash) { + printError(format("fixing missing hash on '%1%'") % i); + info->narHash = current.first; + update = true; + } + + /* Fill in missing narSize fields (from old stores). */ + if (info->narSize == 0) { + printError(format("updating size field on '%1%' to %2%") % i % current.second); + info->narSize = current.second; + update = true; + } + + if (update) { + auto state(_state.lock()); + updatePathInfo(*state, *info); + } + + } + + } catch (Error & e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(i)) + printError(format("error: %1%") % e.msg()); + else + printError(format("warning: %1%") % e.msg()); + errors = true; + } + } + } + + return errors; +} + + +void LocalStore::verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors) +{ + checkInterrupt(); + + if (done.find(path) != done.end()) return; + done.insert(path); + + if (!isStorePath(path)) { + printError(format("path '%1%' is not in the Nix store") % path); + auto state(_state.lock()); + invalidatePath(*state, path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; queryReferrers(path, referrers); + for (auto & i : referrers) + if (i != path) { + verifyPath(i, store, done, validPaths, repair, errors); + if (validPaths.find(i) != validPaths.end()) + canInvalidate = false; + } + + if (canInvalidate) { + printError(format("path '%1%' disappeared, removing from database...") % path); + auto state(_state.lock()); + invalidatePath(*state, path); + } else { + printError(format("path '%1%' disappeared, but it still has valid referrers!") % path); + if (repair) + try { + repairPath(path); + } catch (Error & e) { + printError(format("warning: %1%") % e.msg()); + errors = true; + } + else errors = true; + } + + return; + } + + validPaths.insert(path); +} + + +unsigned int LocalStore::getProtocol() +{ + return PROTOCOL_VERSION; +} + + +#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) + +static void makeMutable(const Path & path) +{ + checkInterrupt(); + + struct stat st = lstat(path); + + if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; + + if (S_ISDIR(st.st_mode)) { + for (auto & i : readDirectory(path)) + makeMutable(path + "/" + i.name); + } + + /* The O_NOFOLLOW is important to prevent us from changing the + mutable bit on the target of a symlink (which would be a + security hole). */ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW | O_CLOEXEC); + if (fd == -1) { + if (errno == ELOOP) return; // it's a symlink + throw SysError(format("opening file '%1%'") % path); + } + + unsigned int flags = 0, old; + + /* Silently ignore errors getting/setting the immutable flag so + that we work correctly on filesystems that don't support it. */ + if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; + old = flags; + flags &= ~FS_IMMUTABLE_FL; + if (old == flags) return; + if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; +} + +/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ +void LocalStore::upgradeStore7() +{ + if (getuid() != 0) return; + printError("removing immutable bits from the Nix store (this may take a while)..."); + makeMutable(realStoreDir); +} + +#else + +void LocalStore::upgradeStore7() +{ +} + +#endif + + +void LocalStore::vacuumDB() +{ + auto state(_state.lock()); + state->db.exec("vacuum"); +} + + +void LocalStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + retrySQLite([&]() { + auto state(_state.lock()); + + SQLiteTxn txn(state->db); + + auto info = std::const_pointer_cast(std::shared_ptr(queryPathInfo(storePath))); + + info->sigs.insert(sigs.begin(), sigs.end()); + + updatePathInfo(*state, *info); + + txn.commit(); + }); +} + + +void LocalStore::signPathInfo(ValidPathInfo & info) +{ + // FIXME: keep secret keys in memory. + + auto secretKeyFiles = settings.secretKeyFiles; + + for (auto & secretKeyFile : secretKeyFiles.get()) { + SecretKey secretKey(readFile(secretKeyFile)); + info.sign(secretKey); + } +} + + +void LocalStore::createUser(const std::string & userName, uid_t userId) +{ + for (auto & dir : { + fmt("%s/profiles/per-user/%s", stateDir, userName), + fmt("%s/gcroots/per-user/%s", stateDir, userName) + }) { + createDirs(dir); + if (chmod(dir.c_str(), 0755) == -1) + throw SysError("changing permissions of directory '%s'", dir); + if (chown(dir.c_str(), userId, getgid()) == -1) + throw SysError("changing owner of directory '%s'", dir); + } +} + + +} diff --git a/third_party/nix/src/libstore/local-store.hh b/third_party/nix/src/libstore/local-store.hh new file mode 100644 index 000000000..379a06af8 --- /dev/null +++ b/third_party/nix/src/libstore/local-store.hh @@ -0,0 +1,322 @@ +#pragma once + +#include "sqlite.hh" + +#include "pathlocks.hh" +#include "store-api.hh" +#include "sync.hh" +#include "util.hh" + +#include +#include +#include +#include + + +namespace nix { + + +/* Nix store and database schema version. Version 1 (or 0) was Nix <= + 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + Nix 1.0. Version 7 is Nix 1.3. Version 10 is 2.0. */ +const int nixSchemaVersion = 10; + + +struct Derivation; + + +struct OptimiseStats +{ + unsigned long filesLinked = 0; + unsigned long long bytesFreed = 0; + unsigned long long blocksFreed = 0; +}; + + +class LocalStore : public LocalFSStore +{ +private: + + /* Lock file used for upgrading. */ + AutoCloseFD globalLock; + + struct State + { + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; + SQLiteStmt stmtAddReference; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; + SQLiteStmt stmtQueryPathFromHashPart; + SQLiteStmt stmtQueryValidPaths; + + /* The file to which we write our temporary roots. */ + AutoCloseFD fdTempRoots; + + /* The last time we checked whether to do an auto-GC, or an + auto-GC finished. */ + std::chrono::time_point lastGCCheck; + + /* Whether auto-GC is running. If so, get gcFuture to wait for + the GC to finish. */ + bool gcRunning = false; + std::shared_future gcFuture; + + /* How much disk space was available after the previous + auto-GC. If the current available disk space is below + minFree but not much below availAfterGC, then there is no + point in starting a new GC. */ + uint64_t availAfterGC = std::numeric_limits::max(); + + std::unique_ptr publicKeys; + }; + + Sync _state; + +public: + + PathSetting realStoreDir_; + + const Path realStoreDir; + const Path dbDir; + const Path linksDir; + const Path reservedPath; + const Path schemaPath; + const Path trashDir; + const Path tempRootsDir; + const Path fnTempRoots; + +private: + + Setting requireSigs{(Store*) this, + settings.requireSigs, + "require-sigs", "whether store paths should have a trusted signature on import"}; + + const PublicKeys & getPublicKeys(); + +public: + + // Hack for build-remote.cc. + PathSet locksHeld = tokenizeString(getEnv("NIX_HELD_LOCKS")); + + /* Initialise the local store, upgrading the schema if + necessary. */ + LocalStore(const Params & params); + + ~LocalStore(); + + /* Implementations of abstract store API methods. */ + + std::string getUri() override; + + bool isValidPathUncached(const Path & path) override; + + PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + PathSet queryAllValidPaths() override; + + void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override; + + void queryReferrers(const Path & path, PathSet & referrers) override; + + PathSet queryValidDerivers(const Path & path) override; + + PathSet queryDerivationOutputs(const Path & path) override; + + StringSet queryDerivationOutputNames(const Path & path) override; + + Path queryPathFromHashPart(const string & hashPart) override; + + PathSet querySubstitutablePaths(const PathSet & paths) override; + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) override; + + Path addToStore(const string & name, const Path & srcPath, + bool recursive, HashType hashAlgo, + PathFilter & filter, RepairFlag repair) override; + + /* Like addToStore(), but the contents of the path are contained + in `dump', which is either a NAR serialisation (if recursive == + true) or simply the contents of a regular file (if recursive == + false). */ + Path addToStoreFromDump(const string & dump, const string & name, + bool recursive = true, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair); + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) override; + + void buildPaths(const PathSet & paths, BuildMode buildMode) override; + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void ensurePath(const Path & path) override; + + void addTempRoot(const Path & path) override; + + void addIndirectRoot(const Path & path) override; + + void syncWithGC() override; + +private: + + typedef std::shared_ptr FDPtr; + typedef list FDs; + + void findTempRoots(FDs & fds, Roots & roots, bool censor); + +public: + + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions & options, GCResults & results) override; + + /* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ + void optimiseStore(OptimiseStats & stats); + + void optimiseStore() override; + + /* Optimise a single store path. */ + void optimisePath(const Path & path); + + bool verifyStore(bool checkContents, RepairFlag repair) override; + + /* Register the validity of a path, i.e., that `path' exists, that + the paths referenced by it exists, and in the case of an output + path of a derivation, that it has been produced by a successful + execution of the derivation (or something equivalent). Also + register the hash of the file system contents of the path. The + hash must be a SHA-256 hash. */ + void registerValidPath(const ValidPathInfo & info); + + void registerValidPaths(const ValidPathInfos & infos); + + unsigned int getProtocol() override; + + void vacuumDB(); + + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + void repairPath(const Path & path); + + void addSignatures(const Path & storePath, const StringSet & sigs) override; + + /* If free disk space in /nix/store if below minFree, delete + garbage until it exceeds maxFree. */ + void autoGC(bool sync = true); + +private: + + int getSchema(); + + void openDB(State & state, bool create); + + void makeStoreWritable(); + + uint64_t queryValidPathId(State & state, const Path & path); + + uint64_t addValidPath(State & state, const ValidPathInfo & info, bool checkOutputs = true); + + void invalidatePath(State & state, const Path & path); + + /* Delete a path from the Nix store. */ + void invalidatePathChecked(const Path & path); + + void verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths, RepairFlag repair, bool & errors); + + void updatePathInfo(State & state, const ValidPathInfo & info); + + void upgradeStore6(); + void upgradeStore7(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path & path); + + struct GCState; + + void deleteGarbage(GCState & state, const Path & path); + + void tryToDelete(GCState & state, const Path & path); + + bool canReachRoot(GCState & state, PathSet & visited, const Path & path); + + void deletePathRecursive(GCState & state, const Path & path); + + bool isActiveTempFile(const GCState & state, + const Path & path, const string & suffix); + + AutoCloseFD openGCLock(LockType lockType); + + void findRoots(const Path & path, unsigned char type, Roots & roots); + + void findRootsNoTemp(Roots & roots, bool censor); + + void findRuntimeRoots(Roots & roots, bool censor); + + void removeUnusedLinks(const GCState & state); + + Path createTempDirInStore(); + + void checkDerivationOutputs(const Path & drvPath, const Derivation & drv); + + typedef std::unordered_set InodeHash; + + InodeHash loadInodeHash(); + Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); + void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash); + + // Internal versions that are not wrapped in retry_sqlite. + bool isValidPath_(State & state, const Path & path); + void queryReferrers(State & state, const Path & path, PathSet & referrers); + + /* Add signatures to a ValidPathInfo using the secret keys + specified by the ‘secret-key-files’ option. */ + void signPathInfo(ValidPathInfo & info); + + Path getRealStoreDir() override { return realStoreDir; } + + void createUser(const std::string & userName, uid_t userId) override; + + friend class DerivationGoal; + friend class SubstitutionGoal; +}; + + +typedef std::pair Inode; +typedef set InodesSeen; + + +/* "Fix", or canonicalise, the meta-data of the files in a store path + after it has been built. In particular: + - the last modification date on each file is set to 1 (i.e., + 00:00:01 1/1/1970 UTC) + - the permissions are set of 444 or 555 (i.e., read-only with or + without execute permission; setuid bits etc. are cleared) + - the owner and group are set to the Nix user and group, if we're + running as root. */ +void canonicalisePathMetaData(const Path & path, uid_t fromUid, InodesSeen & inodesSeen); +void canonicalisePathMetaData(const Path & path, uid_t fromUid); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/third_party/nix/src/libstore/local.mk b/third_party/nix/src/libstore/local.mk new file mode 100644 index 000000000..d690fea28 --- /dev/null +++ b/third_party/nix/src/libstore/local.mk @@ -0,0 +1,60 @@ +libraries += libstore + +libstore_NAME = libnixstore + +libstore_DIR := $(d) + +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) + +libstore_LIBS = libutil + +libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread +ifneq ($(OS), FreeBSD) + libstore_LDFLAGS += -ldl +endif + +libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb + +$(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) + +ifeq ($(ENABLE_S3), 1) + libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core +endif + +ifeq ($(OS), SunOS) + libstore_LDFLAGS += -lsocket +endif + +ifeq ($(HAVE_SECCOMP), 1) + libstore_LDFLAGS += -lseccomp +endif + +libstore_CXXFLAGS = \ + -DNIX_PREFIX=\"$(prefix)\" \ + -DNIX_STORE_DIR=\"$(storedir)\" \ + -DNIX_DATA_DIR=\"$(datadir)\" \ + -DNIX_STATE_DIR=\"$(localstatedir)/nix\" \ + -DNIX_LOG_DIR=\"$(localstatedir)/log/nix\" \ + -DNIX_CONF_DIR=\"$(sysconfdir)/nix\" \ + -DNIX_LIBEXEC_DIR=\"$(libexecdir)\" \ + -DNIX_BIN_DIR=\"$(bindir)\" \ + -DNIX_MAN_DIR=\"$(mandir)\" \ + -DLSOF=\"$(lsof)\" + +ifneq ($(sandbox_shell),) +libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\"" +endif + +$(d)/local-store.cc: $(d)/schema.sql.gen.hh + +$(d)/build.cc: + +%.gen.hh: % + @echo 'R"foo(' >> $@.tmp + $(trace-gen) cat $< >> $@.tmp + @echo ')foo"' >> $@.tmp + @mv $@.tmp $@ + +clean-files += $(d)/schema.sql.gen.hh + +$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) diff --git a/third_party/nix/src/libstore/machines.cc b/third_party/nix/src/libstore/machines.cc new file mode 100644 index 000000000..f848582da --- /dev/null +++ b/third_party/nix/src/libstore/machines.cc @@ -0,0 +1,100 @@ +#include "machines.hh" +#include "util.hh" +#include "globals.hh" + +#include + +namespace nix { + +Machine::Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey) : + storeUri( + // Backwards compatibility: if the URI is a hostname, + // prepend ssh://. + storeUri.find("://") != std::string::npos + || hasPrefix(storeUri, "local") + || hasPrefix(storeUri, "remote") + || hasPrefix(storeUri, "auto") + || hasPrefix(storeUri, "/") + ? storeUri + : "ssh://" + storeUri), + systemTypes(systemTypes), + sshKey(sshKey), + maxJobs(maxJobs), + speedFactor(std::max(1U, speedFactor)), + supportedFeatures(supportedFeatures), + mandatoryFeatures(mandatoryFeatures), + sshPublicHostKey(sshPublicHostKey) +{} + +bool Machine::allSupported(const std::set & features) const { + return std::all_of(features.begin(), features.end(), + [&](const string & feature) { + return supportedFeatures.count(feature) || + mandatoryFeatures.count(feature); + }); +} + +bool Machine::mandatoryMet(const std::set & features) const { + return std::all_of(mandatoryFeatures.begin(), mandatoryFeatures.end(), + [&](const string & feature) { + return features.count(feature); + }); +} + +void parseMachines(const std::string & s, Machines & machines) +{ + for (auto line : tokenizeString>(s, "\n;")) { + trim(line); + line.erase(std::find(line.begin(), line.end(), '#'), line.end()); + if (line.empty()) continue; + + if (line[0] == '@') { + auto file = trim(std::string(line, 1)); + try { + parseMachines(readFile(file), machines); + } catch (const SysError & e) { + if (e.errNo != ENOENT) + throw; + debug("cannot find machines file '%s'", file); + } + continue; + } + + auto tokens = tokenizeString>(line); + auto sz = tokens.size(); + if (sz < 1) + throw FormatError("bad machine specification '%s'", line); + + auto isSet = [&](size_t n) { + return tokens.size() > n && tokens[n] != "" && tokens[n] != "-"; + }; + + machines.emplace_back(tokens[0], + isSet(1) ? tokenizeString>(tokens[1], ",") : std::vector{settings.thisSystem}, + isSet(2) ? tokens[2] : "", + isSet(3) ? std::stoull(tokens[3]) : 1LL, + isSet(4) ? std::stoull(tokens[4]) : 1LL, + isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + isSet(7) ? tokens[7] : ""); + } +} + +Machines getMachines() +{ + static auto machines = [&]() { + Machines machines; + parseMachines(settings.builders, machines); + return machines; + }(); + return machines; +} + +} diff --git a/third_party/nix/src/libstore/machines.hh b/third_party/nix/src/libstore/machines.hh new file mode 100644 index 000000000..de92eb924 --- /dev/null +++ b/third_party/nix/src/libstore/machines.hh @@ -0,0 +1,39 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +struct Machine { + + const string storeUri; + const std::vector systemTypes; + const string sshKey; + const unsigned int maxJobs; + const unsigned int speedFactor; + const std::set supportedFeatures; + const std::set mandatoryFeatures; + const std::string sshPublicHostKey; + bool enabled = true; + + bool allSupported(const std::set & features) const; + + bool mandatoryMet(const std::set & features) const; + + Machine(decltype(storeUri) storeUri, + decltype(systemTypes) systemTypes, + decltype(sshKey) sshKey, + decltype(maxJobs) maxJobs, + decltype(speedFactor) speedFactor, + decltype(supportedFeatures) supportedFeatures, + decltype(mandatoryFeatures) mandatoryFeatures, + decltype(sshPublicHostKey) sshPublicHostKey); +}; + +typedef std::vector Machines; + +void parseMachines(const std::string & s, Machines & machines); + +Machines getMachines(); + +} diff --git a/third_party/nix/src/libstore/misc.cc b/third_party/nix/src/libstore/misc.cc new file mode 100644 index 000000000..dddf13430 --- /dev/null +++ b/third_party/nix/src/libstore/misc.cc @@ -0,0 +1,282 @@ +#include "derivations.hh" +#include "parsed-derivations.hh" +#include "globals.hh" +#include "local-store.hh" +#include "store-api.hh" +#include "thread-pool.hh" + + +namespace nix { + + +void Store::computeFSClosure(const PathSet & startPaths, + PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +{ + struct State + { + size_t pending; + PathSet & paths; + std::exception_ptr exc; + }; + + Sync state_(State{0, paths_, 0}); + + std::function enqueue; + + std::condition_variable done; + + enqueue = [&](const Path & path) -> void { + { + auto state(state_.lock()); + if (state->exc) return; + if (state->paths.count(path)) return; + state->paths.insert(path); + state->pending++; + } + + queryPathInfo(path, {[&, path](std::future> fut) { + // FIXME: calls to isValidPath() should be async + + try { + auto info = fut.get(); + + if (flipDirection) { + + PathSet referrers; + queryReferrers(path, referrers); + for (auto & ref : referrers) + if (ref != path) + enqueue(ref); + + if (includeOutputs) + for (auto & i : queryValidDerivers(path)) + enqueue(i); + + if (includeDerivers && isDerivation(path)) + for (auto & i : queryDerivationOutputs(path)) + if (isValidPath(i) && queryPathInfo(i)->deriver == path) + enqueue(i); + + } else { + + for (auto & ref : info->references) + if (ref != path) + enqueue(ref); + + if (includeOutputs && isDerivation(path)) + for (auto & i : queryDerivationOutputs(path)) + if (isValidPath(i)) enqueue(i); + + if (includeDerivers && isValidPath(info->deriver)) + enqueue(info->deriver); + + } + + { + auto state(state_.lock()); + assert(state->pending); + if (!--state->pending) done.notify_one(); + } + + } catch (...) { + auto state(state_.lock()); + if (!state->exc) state->exc = std::current_exception(); + assert(state->pending); + if (!--state->pending) done.notify_one(); + }; + }}); + }; + + for (auto & startPath : startPaths) + enqueue(startPath); + + { + auto state(state_.lock()); + while (state->pending) state.wait(done); + if (state->exc) std::rethrow_exception(state->exc); + } +} + + +void Store::computeFSClosure(const Path & startPath, + PathSet & paths_, bool flipDirection, bool includeOutputs, bool includeDerivers) +{ + computeFSClosure(PathSet{startPath}, paths_, flipDirection, includeOutputs, includeDerivers); +} + + +void Store::queryMissing(const PathSet & targets, + PathSet & willBuild_, PathSet & willSubstitute_, PathSet & unknown_, + unsigned long long & downloadSize_, unsigned long long & narSize_) +{ + Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths"); + + downloadSize_ = narSize_ = 0; + + ThreadPool pool; + + struct State + { + PathSet done; + PathSet & unknown, & willSubstitute, & willBuild; + unsigned long long & downloadSize; + unsigned long long & narSize; + }; + + struct DrvState + { + size_t left; + bool done = false; + PathSet outPaths; + DrvState(size_t left) : left(left) { } + }; + + Sync state_(State{PathSet(), unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_}); + + std::function doPath; + + auto mustBuildDrv = [&](const Path & drvPath, const Derivation & drv) { + { + auto state(state_.lock()); + state->willBuild.insert(drvPath); + } + + for (auto & i : drv.inputDrvs) + pool.enqueue(std::bind(doPath, makeDrvPathWithOutputs(i.first, i.second))); + }; + + auto checkOutput = [&]( + const Path & drvPath, ref drv, const Path & outPath, ref> drvState_) + { + if (drvState_->lock()->done) return; + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({outPath}, infos); + + if (infos.empty()) { + drvState_->lock()->done = true; + mustBuildDrv(drvPath, *drv); + } else { + { + auto drvState(drvState_->lock()); + if (drvState->done) return; + assert(drvState->left); + drvState->left--; + drvState->outPaths.insert(outPath); + if (!drvState->left) { + for (auto & path : drvState->outPaths) + pool.enqueue(std::bind(doPath, path)); + } + } + } + }; + + doPath = [&](const Path & path) { + + { + auto state(state_.lock()); + if (state->done.count(path)) return; + state->done.insert(path); + } + + DrvPathWithOutputs i2 = parseDrvPathWithOutputs(path); + + if (isDerivation(i2.first)) { + if (!isValidPath(i2.first)) { + // FIXME: we could try to substitute the derivation. + auto state(state_.lock()); + state->unknown.insert(path); + return; + } + + Derivation drv = derivationFromPath(i2.first); + ParsedDerivation parsedDrv(i2.first, drv); + + PathSet invalid; + for (auto & j : drv.outputs) + if (wantOutput(j.first, i2.second) + && !isValidPath(j.second.path)) + invalid.insert(j.second.path); + if (invalid.empty()) return; + + if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) { + auto drvState = make_ref>(DrvState(invalid.size())); + for (auto & output : invalid) + pool.enqueue(std::bind(checkOutput, i2.first, make_ref(drv), output, drvState)); + } else + mustBuildDrv(i2.first, drv); + + } else { + + if (isValidPath(path)) return; + + SubstitutablePathInfos infos; + querySubstitutablePathInfos({path}, infos); + + if (infos.empty()) { + auto state(state_.lock()); + state->unknown.insert(path); + return; + } + + auto info = infos.find(path); + assert(info != infos.end()); + + { + auto state(state_.lock()); + state->willSubstitute.insert(path); + state->downloadSize += info->second.downloadSize; + state->narSize += info->second.narSize; + } + + for (auto & ref : info->second.references) + pool.enqueue(std::bind(doPath, ref)); + } + }; + + for (auto & path : targets) + pool.enqueue(std::bind(doPath, path)); + + pool.process(); +} + + +Paths Store::topoSortPaths(const PathSet & paths) +{ + Paths sorted; + PathSet visited, parents; + + std::function dfsVisit; + + dfsVisit = [&](const Path & path, const Path * parent) { + if (parents.find(path) != parents.end()) + throw BuildError(format("cycle detected in the references of '%1%' from '%2%'") % path % *parent); + + if (visited.find(path) != visited.end()) return; + visited.insert(path); + parents.insert(path); + + PathSet references; + try { + references = queryPathInfo(path)->references; + } catch (InvalidPath &) { + } + + for (auto & i : references) + /* Don't traverse into paths that don't exist. That can + happen due to substitutes for non-existent paths. */ + if (i != path && paths.find(i) != paths.end()) + dfsVisit(i, &path); + + sorted.push_front(path); + parents.erase(path); + }; + + for (auto & i : paths) + dfsVisit(i, nullptr); + + return sorted; +} + + +} diff --git a/third_party/nix/src/libstore/nar-accessor.cc b/third_party/nix/src/libstore/nar-accessor.cc new file mode 100644 index 000000000..b74480684 --- /dev/null +++ b/third_party/nix/src/libstore/nar-accessor.cc @@ -0,0 +1,266 @@ +#include "nar-accessor.hh" +#include "archive.hh" +#include "json.hh" + +#include +#include +#include + +#include + +namespace nix { + +struct NarMember +{ + FSAccessor::Type type = FSAccessor::Type::tMissing; + + bool isExecutable = false; + + /* If this is a regular file, position of the contents of this + file in the NAR. */ + size_t start = 0, size = 0; + + std::string target; + + /* If this is a directory, all the children of the directory. */ + std::map children; +}; + +struct NarAccessor : public FSAccessor +{ + std::shared_ptr nar; + + GetNarBytes getNarBytes; + + NarMember root; + + struct NarIndexer : ParseSink, StringSource + { + NarAccessor & acc; + + std::stack parents; + + std::string currentStart; + bool isExec = false; + + NarIndexer(NarAccessor & acc, const std::string & nar) + : StringSource(nar), acc(acc) + { } + + void createMember(const Path & path, NarMember member) { + size_t level = std::count(path.begin(), path.end(), '/'); + while (parents.size() > level) parents.pop(); + + if (parents.empty()) { + acc.root = std::move(member); + parents.push(&acc.root); + } else { + if (parents.top()->type != FSAccessor::Type::tDirectory) + throw Error("NAR file missing parent directory of path '%s'", path); + auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); + parents.push(&result.first->second); + } + } + + void createDirectory(const Path & path) override + { + createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + } + + void createRegularFile(const Path & path) override + { + createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + } + + void isExecutable() override + { + parents.top()->isExecutable = true; + } + + void preallocateContents(unsigned long long size) override + { + currentStart = string(s, pos, 16); + assert(size <= std::numeric_limits::max()); + parents.top()->size = (size_t)size; + parents.top()->start = pos; + } + + void receiveContents(unsigned char * data, unsigned int len) override + { + // Sanity check + if (!currentStart.empty()) { + assert(len < 16 || currentStart == string((char *) data, 16)); + currentStart.clear(); + } + } + + void createSymlink(const Path & path, const string & target) override + { + createMember(path, + NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + } + }; + + NarAccessor(ref nar) : nar(nar) + { + NarIndexer indexer(*this, *nar); + parseDump(indexer, indexer); + } + + NarAccessor(const std::string & listing, GetNarBytes getNarBytes) + : getNarBytes(getNarBytes) + { + using json = nlohmann::json; + + std::function recurse; + + recurse = [&](NarMember & member, json & v) { + std::string type = v["type"]; + + if (type == "directory") { + member.type = FSAccessor::Type::tDirectory; + for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { + std::string name = i.key(); + recurse(member.children[name], i.value()); + } + } else if (type == "regular") { + member.type = FSAccessor::Type::tRegular; + member.size = v["size"]; + member.isExecutable = v.value("executable", false); + member.start = v["narOffset"]; + } else if (type == "symlink") { + member.type = FSAccessor::Type::tSymlink; + member.target = v.value("target", ""); + } else return; + }; + + json v = json::parse(listing); + recurse(root, v); + } + + NarMember * find(const Path & path) + { + Path canon = path == "" ? "" : canonPath(path); + NarMember * current = &root; + auto end = path.end(); + for (auto it = path.begin(); it != end; ) { + // because it != end, the remaining component is non-empty so we need + // a directory + if (current->type != FSAccessor::Type::tDirectory) return nullptr; + + // skip slash (canonPath above ensures that this is always a slash) + assert(*it == '/'); + it += 1; + + // lookup current component + auto next = std::find(it, end, '/'); + auto child = current->children.find(std::string(it, next)); + if (child == current->children.end()) return nullptr; + current = &child->second; + + it = next; + } + + return current; + } + + NarMember & get(const Path & path) { + auto result = find(path); + if (result == nullptr) + throw Error("NAR file does not contain path '%1%'", path); + return *result; + } + + Stat stat(const Path & path) override + { + auto i = find(path); + if (i == nullptr) + return {FSAccessor::Type::tMissing, 0, false}; + return {i->type, i->size, i->isExecutable, i->start}; + } + + StringSet readDirectory(const Path & path) override + { + auto i = get(path); + + if (i.type != FSAccessor::Type::tDirectory) + throw Error(format("path '%1%' inside NAR file is not a directory") % path); + + StringSet res; + for (auto & child : i.children) + res.insert(child.first); + + return res; + } + + std::string readFile(const Path & path) override + { + auto i = get(path); + if (i.type != FSAccessor::Type::tRegular) + throw Error(format("path '%1%' inside NAR file is not a regular file") % path); + + if (getNarBytes) return getNarBytes(i.start, i.size); + + assert(nar); + return std::string(*nar, i.start, i.size); + } + + std::string readLink(const Path & path) override + { + auto i = get(path); + if (i.type != FSAccessor::Type::tSymlink) + throw Error(format("path '%1%' inside NAR file is not a symlink") % path); + return i.target; + } +}; + +ref makeNarAccessor(ref nar) +{ + return make_ref(nar); +} + +ref makeLazyNarAccessor(const std::string & listing, + GetNarBytes getNarBytes) +{ + return make_ref(listing, getNarBytes); +} + +void listNar(JSONPlaceholder & res, ref accessor, + const Path & path, bool recurse) +{ + auto st = accessor->stat(path); + + auto obj = res.object(); + + switch (st.type) { + case FSAccessor::Type::tRegular: + obj.attr("type", "regular"); + obj.attr("size", st.fileSize); + if (st.isExecutable) + obj.attr("executable", true); + if (st.narOffset) + obj.attr("narOffset", st.narOffset); + break; + case FSAccessor::Type::tDirectory: + obj.attr("type", "directory"); + { + auto res2 = obj.object("entries"); + for (auto & name : accessor->readDirectory(path)) { + if (recurse) { + auto res3 = res2.placeholder(name); + listNar(res3, accessor, path + "/" + name, true); + } else + res2.object(name); + } + } + break; + case FSAccessor::Type::tSymlink: + obj.attr("type", "symlink"); + obj.attr("target", accessor->readLink(path)); + break; + default: + throw Error("path '%s' does not exist in NAR", path); + } +} + +} diff --git a/third_party/nix/src/libstore/nar-accessor.hh b/third_party/nix/src/libstore/nar-accessor.hh new file mode 100644 index 000000000..2871199de --- /dev/null +++ b/third_party/nix/src/libstore/nar-accessor.hh @@ -0,0 +1,30 @@ +#pragma once + +#include + +#include "fs-accessor.hh" + +namespace nix { + +/* Return an object that provides access to the contents of a NAR + file. */ +ref makeNarAccessor(ref nar); + +/* Create a NAR accessor from a NAR listing (in the format produced by + listNar()). The callback getNarBytes(offset, length) is used by the + readFile() method of the accessor to get the contents of files + inside the NAR. */ +typedef std::function GetNarBytes; + +ref makeLazyNarAccessor( + const std::string & listing, + GetNarBytes getNarBytes); + +class JSONPlaceholder; + +/* Write a JSON representation of the contents of a NAR (except file + contents). */ +void listNar(JSONPlaceholder & res, ref accessor, + const Path & path, bool recurse); + +} diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.cc b/third_party/nix/src/libstore/nar-info-disk-cache.cc new file mode 100644 index 000000000..32ad7f2b2 --- /dev/null +++ b/third_party/nix/src/libstore/nar-info-disk-cache.cc @@ -0,0 +1,267 @@ +#include "nar-info-disk-cache.hh" +#include "sync.hh" +#include "sqlite.hh" +#include "globals.hh" + +#include + +namespace nix { + +static const char * schema = R"sql( + +create table if not exists BinaryCaches ( + id integer primary key autoincrement not null, + url text unique not null, + timestamp integer not null, + storeDir text not null, + wantMassQuery integer not null, + priority integer not null +); + +create table if not exists NARs ( + cache integer not null, + hashPart text not null, + namePart text, + url text, + compression text, + fileHash text, + fileSize integer, + narHash text, + narSize integer, + refs text, + deriver text, + sigs text, + ca text, + timestamp integer not null, + present integer not null, + primary key (cache, hashPart), + foreign key (cache) references BinaryCaches(id) on delete cascade +); + +create table if not exists LastPurge ( + dummy text primary key, + value integer +); + +)sql"; + +class NarInfoDiskCacheImpl : public NarInfoDiskCache +{ +public: + + /* How often to purge expired entries from the cache. */ + const int purgeInterval = 24 * 3600; + + struct Cache + { + int id; + Path storeDir; + bool wantMassQuery; + int priority; + }; + + struct State + { + SQLite db; + SQLiteStmt insertCache, queryCache, insertNAR, insertMissingNAR, queryNAR, purgeCache; + std::map caches; + }; + + Sync _state; + + NarInfoDiskCacheImpl() + { + auto state(_state.lock()); + + Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + + if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK) + throwSQLiteError(state->db, "setting timeout"); + + // We can always reproduce the cache. + state->db.exec("pragma synchronous = off"); + state->db.exec("pragma main.journal_mode = truncate"); + + state->db.exec(schema); + + state->insertCache.create(state->db, + "insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)"); + + state->queryCache.create(state->db, + "select id, storeDir, wantMassQuery, priority from BinaryCaches where url = ?"); + + state->insertNAR.create(state->db, + "insert or replace into NARs(cache, hashPart, namePart, url, compression, fileHash, fileSize, narHash, " + "narSize, refs, deriver, sigs, ca, timestamp, present) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1)"); + + state->insertMissingNAR.create(state->db, + "insert or replace into NARs(cache, hashPart, timestamp, present) values (?, ?, ?, 0)"); + + state->queryNAR.create(state->db, + "select present, namePart, url, compression, fileHash, fileSize, narHash, narSize, refs, deriver, sigs, ca from NARs where cache = ? and hashPart = ? and ((present = 0 and timestamp > ?) or (present = 1 and timestamp > ?))"); + + /* Periodically purge expired entries from the database. */ + retrySQLite([&]() { + auto now = time(0); + + SQLiteStmt queryLastPurge(state->db, "select value from LastPurge"); + auto queryLastPurge_(queryLastPurge.use()); + + if (!queryLastPurge_.next() || queryLastPurge_.getInt(0) < now - purgeInterval) { + SQLiteStmt(state->db, + "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") + .use() + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache) + .exec(); + + debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); + + SQLiteStmt(state->db, + "insert or replace into LastPurge(dummy, value) values ('', ?)") + .use()(now).exec(); + } + }); + } + + Cache & getCache(State & state, const std::string & uri) + { + auto i = state.caches.find(uri); + if (i == state.caches.end()) abort(); + return i->second; + } + + void createCache(const std::string & uri, const Path & storeDir, bool wantMassQuery, int priority) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + // FIXME: race + + state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority).exec(); + assert(sqlite3_changes(state->db) == 1); + state->caches[uri] = Cache{(int) sqlite3_last_insert_rowid(state->db), storeDir, wantMassQuery, priority}; + }); + } + + bool cacheExists(const std::string & uri, + bool & wantMassQuery, int & priority) override + { + return retrySQLite([&]() { + auto state(_state.lock()); + + auto i = state->caches.find(uri); + if (i == state->caches.end()) { + auto queryCache(state->queryCache.use()(uri)); + if (!queryCache.next()) return false; + state->caches.emplace(uri, + Cache{(int) queryCache.getInt(0), queryCache.getStr(1), queryCache.getInt(2) != 0, (int) queryCache.getInt(3)}); + } + + auto & cache(getCache(*state, uri)); + + wantMassQuery = cache.wantMassQuery; + priority = cache.priority; + + return true; + }); + } + + std::pair> lookupNarInfo( + const std::string & uri, const std::string & hashPart) override + { + return retrySQLite>>( + [&]() -> std::pair> { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + auto now = time(0); + + auto queryNAR(state->queryNAR.use() + (cache.id) + (hashPart) + (now - settings.ttlNegativeNarInfoCache) + (now - settings.ttlPositiveNarInfoCache)); + + if (!queryNAR.next()) + return {oUnknown, 0}; + + if (!queryNAR.getInt(0)) + return {oInvalid, 0}; + + auto narInfo = make_ref(); + + auto namePart = queryNAR.getStr(1); + narInfo->path = cache.storeDir + "/" + + hashPart + (namePart.empty() ? "" : "-" + namePart); + narInfo->url = queryNAR.getStr(2); + narInfo->compression = queryNAR.getStr(3); + if (!queryNAR.isNull(4)) + narInfo->fileHash = Hash(queryNAR.getStr(4)); + narInfo->fileSize = queryNAR.getInt(5); + narInfo->narHash = Hash(queryNAR.getStr(6)); + narInfo->narSize = queryNAR.getInt(7); + for (auto & r : tokenizeString(queryNAR.getStr(8), " ")) + narInfo->references.insert(cache.storeDir + "/" + r); + if (!queryNAR.isNull(9)) + narInfo->deriver = cache.storeDir + "/" + queryNAR.getStr(9); + for (auto & sig : tokenizeString(queryNAR.getStr(10), " ")) + narInfo->sigs.insert(sig); + narInfo->ca = queryNAR.getStr(11); + + return {oValid, narInfo}; + }); + } + + void upsertNarInfo( + const std::string & uri, const std::string & hashPart, + std::shared_ptr info) override + { + retrySQLite([&]() { + auto state(_state.lock()); + + auto & cache(getCache(*state, uri)); + + if (info) { + + auto narInfo = std::dynamic_pointer_cast(info); + + assert(hashPart == storePathToHash(info->path)); + + state->insertNAR.use() + (cache.id) + (hashPart) + (storePathToName(info->path)) + (narInfo ? narInfo->url : "", narInfo != 0) + (narInfo ? narInfo->compression : "", narInfo != 0) + (narInfo && narInfo->fileHash ? narInfo->fileHash.to_string() : "", narInfo && narInfo->fileHash) + (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) + (info->narHash.to_string()) + (info->narSize) + (concatStringsSep(" ", info->shortRefs())) + (info->deriver != "" ? baseNameOf(info->deriver) : "", info->deriver != "") + (concatStringsSep(" ", info->sigs)) + (info->ca) + (time(0)).exec(); + + } else { + state->insertMissingNAR.use() + (cache.id) + (hashPart) + (time(0)).exec(); + } + }); + } +}; + +ref getNarInfoDiskCache() +{ + static ref cache = make_ref(); + return cache; +} + +} diff --git a/third_party/nix/src/libstore/nar-info-disk-cache.hh b/third_party/nix/src/libstore/nar-info-disk-cache.hh new file mode 100644 index 000000000..88d909732 --- /dev/null +++ b/third_party/nix/src/libstore/nar-info-disk-cache.hh @@ -0,0 +1,31 @@ +#pragma once + +#include "ref.hh" +#include "nar-info.hh" + +namespace nix { + +class NarInfoDiskCache +{ +public: + typedef enum { oValid, oInvalid, oUnknown } Outcome; + + virtual void createCache(const std::string & uri, const Path & storeDir, + bool wantMassQuery, int priority) = 0; + + virtual bool cacheExists(const std::string & uri, + bool & wantMassQuery, int & priority) = 0; + + virtual std::pair> lookupNarInfo( + const std::string & uri, const std::string & hashPart) = 0; + + virtual void upsertNarInfo( + const std::string & uri, const std::string & hashPart, + std::shared_ptr info) = 0; +}; + +/* Return a singleton cache object that can be used concurrently by + multiple threads. */ +ref getNarInfoDiskCache(); + +} diff --git a/third_party/nix/src/libstore/nar-info.cc b/third_party/nix/src/libstore/nar-info.cc new file mode 100644 index 000000000..cb568ccdc --- /dev/null +++ b/third_party/nix/src/libstore/nar-info.cc @@ -0,0 +1,116 @@ +#include "globals.hh" +#include "nar-info.hh" + +namespace nix { + +NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) +{ + auto corrupt = [&]() { + throw Error(format("NAR info file '%1%' is corrupt") % whence); + }; + + auto parseHashField = [&](const string & s) { + try { + return Hash(s); + } catch (BadHash &) { + corrupt(); + return Hash(); // never reached + } + }; + + size_t pos = 0; + while (pos < s.size()) { + + size_t colon = s.find(':', pos); + if (colon == std::string::npos) corrupt(); + + std::string name(s, pos, colon - pos); + + size_t eol = s.find('\n', colon + 2); + if (eol == std::string::npos) corrupt(); + + std::string value(s, colon + 2, eol - colon - 2); + + if (name == "StorePath") { + if (!store.isStorePath(value)) corrupt(); + path = value; + } + else if (name == "URL") + url = value; + else if (name == "Compression") + compression = value; + else if (name == "FileHash") + fileHash = parseHashField(value); + else if (name == "FileSize") { + if (!string2Int(value, fileSize)) corrupt(); + } + else if (name == "NarHash") + narHash = parseHashField(value); + else if (name == "NarSize") { + if (!string2Int(value, narSize)) corrupt(); + } + else if (name == "References") { + auto refs = tokenizeString(value, " "); + if (!references.empty()) corrupt(); + for (auto & r : refs) { + auto r2 = store.storeDir + "/" + r; + if (!store.isStorePath(r2)) corrupt(); + references.insert(r2); + } + } + else if (name == "Deriver") { + if (value != "unknown-deriver") { + auto p = store.storeDir + "/" + value; + if (!store.isStorePath(p)) corrupt(); + deriver = p; + } + } + else if (name == "System") + system = value; + else if (name == "Sig") + sigs.insert(value); + else if (name == "CA") { + if (!ca.empty()) corrupt(); + ca = value; + } + + pos = eol + 1; + } + + if (compression == "") compression = "bzip2"; + + if (path.empty() || url.empty() || narSize == 0 || !narHash) corrupt(); +} + +std::string NarInfo::to_string() const +{ + std::string res; + res += "StorePath: " + path + "\n"; + res += "URL: " + url + "\n"; + assert(compression != ""); + res += "Compression: " + compression + "\n"; + assert(fileHash.type == htSHA256); + res += "FileHash: " + fileHash.to_string(Base32) + "\n"; + res += "FileSize: " + std::to_string(fileSize) + "\n"; + assert(narHash.type == htSHA256); + res += "NarHash: " + narHash.to_string(Base32) + "\n"; + res += "NarSize: " + std::to_string(narSize) + "\n"; + + res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; + + if (!deriver.empty()) + res += "Deriver: " + baseNameOf(deriver) + "\n"; + + if (!system.empty()) + res += "System: " + system + "\n"; + + for (auto sig : sigs) + res += "Sig: " + sig + "\n"; + + if (!ca.empty()) + res += "CA: " + ca + "\n"; + + return res; +} + +} diff --git a/third_party/nix/src/libstore/nar-info.hh b/third_party/nix/src/libstore/nar-info.hh new file mode 100644 index 000000000..4995061fb --- /dev/null +++ b/third_party/nix/src/libstore/nar-info.hh @@ -0,0 +1,24 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "store-api.hh" + +namespace nix { + +struct NarInfo : ValidPathInfo +{ + std::string url; + std::string compression; + Hash fileHash; + uint64_t fileSize = 0; + std::string system; + + NarInfo() { } + NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } + NarInfo(const Store & store, const std::string & s, const std::string & whence); + + std::string to_string() const; +}; + +} diff --git a/third_party/nix/src/libstore/nix-store.pc.in b/third_party/nix/src/libstore/nix-store.pc.in new file mode 100644 index 000000000..6d67b1e03 --- /dev/null +++ b/third_party/nix/src/libstore/nix-store.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixstore -lnixutil +Cflags: -I${includedir}/nix -std=c++17 diff --git a/third_party/nix/src/libstore/optimise-store.cc b/third_party/nix/src/libstore/optimise-store.cc new file mode 100644 index 000000000..991512f21 --- /dev/null +++ b/third_party/nix/src/libstore/optimise-store.cc @@ -0,0 +1,302 @@ +#include "util.hh" +#include "local-store.hh" +#include "globals.hh" + +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace nix { + + +static void makeWritable(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + if (chmod(path.c_str(), st.st_mode | S_IWUSR) == -1) + throw SysError(format("changing writability of '%1%'") % path); +} + + +struct MakeReadOnly +{ + Path path; + MakeReadOnly(const Path & path) : path(path) { } + ~MakeReadOnly() + { + try { + /* This will make the path read-only. */ + if (path != "") canonicaliseTimestampAndPermissions(path); + } catch (...) { + ignoreException(); + } + } +}; + + +LocalStore::InodeHash LocalStore::loadInodeHash() +{ + debug("loading hash inodes in memory"); + InodeHash inodeHash; + + AutoCloseDir dir(opendir(linksDir.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % linksDir); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + // We don't care if we hit non-hash files, anything goes + inodeHash.insert(dirent->d_ino); + } + if (errno) throw SysError(format("reading directory '%1%'") % linksDir); + + printMsg(lvlTalkative, format("loaded %1% hash inodes") % inodeHash.size()); + + return inodeHash; +} + + +Strings LocalStore::readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash) +{ + Strings names; + + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + + if (inodeHash.count(dirent->d_ino)) { + debug(format("'%1%' is already linked") % dirent->d_name); + continue; + } + + string name = dirent->d_name; + if (name == "." || name == "..") continue; + names.push_back(name); + } + if (errno) throw SysError(format("reading directory '%1%'") % path); + + return names; +} + + +void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, + const Path & path, InodeHash & inodeHash) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + +#if __APPLE__ + /* HFS/macOS has some undocumented security feature disabling hardlinking for + special files within .app dirs. *.app/Contents/PkgInfo and + *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See + https://github.com/NixOS/nix/issues/1443 for more discussion. */ + + if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) + { + debug(format("'%1%' is not allowed to be linked in macOS") % path); + return; + } +#endif + + if (S_ISDIR(st.st_mode)) { + Strings names = readDirectoryIgnoringInodes(path, inodeHash); + for (auto & i : names) + optimisePath_(act, stats, path + "/" + i, inodeHash); + return; + } + + /* We can hard link regular files and maybe symlinks. */ + if (!S_ISREG(st.st_mode) +#if CAN_LINK_SYMLINK + && !S_ISLNK(st.st_mode) +#endif + ) return; + + /* Sometimes SNAFUs can cause files in the Nix store to be + modified, in particular when running programs as root under + NixOS (example: $fontconfig/var/cache being modified). Skip + those files. FIXME: check the modification time. */ + if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) { + printError(format("skipping suspicious writable file '%1%'") % path); + return; + } + + /* This can still happen on top-level files. */ + if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) { + debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2)); + return; + } + + /* Hash the file. Note that hashPath() returns the hash over the + NAR serialisation, which includes the execute bit on the file. + Thus, executable and non-executable files with the same + contents *won't* be linked (which is good because otherwise the + permissions would be screwed up). + + Also note that if `path' is a symlink, then we're hashing the + contents of the symlink (i.e. the result of readlink()), not + the contents of the target (which may not even exist). */ + Hash hash = hashPath(htSHA256, path).first; + debug(format("'%1%' has hash '%2%'") % path % hash.to_string()); + + /* Check if this is a known hash. */ + Path linkPath = linksDir + "/" + hash.to_string(Base32, false); + + retry: + if (!pathExists(linkPath)) { + /* Nope, create a hard link in the links directory. */ + if (link(path.c_str(), linkPath.c_str()) == 0) { + inodeHash.insert(st.st_ino); + return; + } + + switch (errno) { + case EEXIST: + /* Fall through if another process created ‘linkPath’ before + we did. */ + break; + + case ENOSPC: + /* On ext4, that probably means the directory index is + full. When that happens, it's fine to ignore it: we + just effectively disable deduplication of this + file. */ + printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); + return; + + default: + throw SysError("cannot link '%1%' to '%2%'", linkPath, path); + } + } + + /* Yes! We've seen a file with the same contents. Replace the + current file with a hard link to that file. */ + struct stat stLink; + if (lstat(linkPath.c_str(), &stLink)) + throw SysError(format("getting attributes of path '%1%'") % linkPath); + + if (st.st_ino == stLink.st_ino) { + debug(format("'%1%' is already linked to '%2%'") % path % linkPath); + return; + } + + if (st.st_size != stLink.st_size) { + printError(format("removing corrupted link '%1%'") % linkPath); + unlink(linkPath.c_str()); + goto retry; + } + + printMsg(lvlTalkative, format("linking '%1%' to '%2%'") % path % linkPath); + + /* Make the containing directory writable, but only if it's not + the store itself (we don't want or need to mess with its + permissions). */ + bool mustToggle = dirOf(path) != realStoreDir; + if (mustToggle) makeWritable(dirOf(path)); + + /* When we're done, make the directory read-only again and reset + its timestamp back to 0. */ + MakeReadOnly makeReadOnly(mustToggle ? dirOf(path) : ""); + + Path tempLink = (format("%1%/.tmp-link-%2%-%3%") + % realStoreDir % getpid() % random()).str(); + + if (link(linkPath.c_str(), tempLink.c_str()) == -1) { + if (errno == EMLINK) { + /* Too many links to the same file (>= 32000 on most file + systems). This is likely to happen with empty files. + Just shrug and ignore. */ + if (st.st_size) + printInfo(format("'%1%' has maximum number of links") % linkPath); + return; + } + throw SysError("cannot link '%1%' to '%2%'", tempLink, linkPath); + } + + /* Atomically replace the old file with the new hard link. */ + if (rename(tempLink.c_str(), path.c_str()) == -1) { + if (unlink(tempLink.c_str()) == -1) + printError(format("unable to unlink '%1%'") % tempLink); + if (errno == EMLINK) { + /* Some filesystems generate too many links on the rename, + rather than on the original link. (Probably it + temporarily increases the st_nlink field before + decreasing it again.) */ + debug("'%s' has reached maximum number of links", linkPath); + return; + } + throw SysError(format("cannot rename '%1%' to '%2%'") % tempLink % path); + } + + stats.filesLinked++; + stats.bytesFreed += st.st_size; + stats.blocksFreed += st.st_blocks; + + if (act) + act->result(resFileLinked, st.st_size, st.st_blocks); +} + + +void LocalStore::optimiseStore(OptimiseStats & stats) +{ + Activity act(*logger, actOptimiseStore); + + PathSet paths = queryAllValidPaths(); + InodeHash inodeHash = loadInodeHash(); + + act.progress(0, paths.size()); + + uint64_t done = 0; + + for (auto & i : paths) { + addTempRoot(i); + if (!isValidPath(i)) continue; /* path was GC'ed, probably */ + { + Activity act(*logger, lvlTalkative, actUnknown, fmt("optimising path '%s'", i)); + optimisePath_(&act, stats, realStoreDir + "/" + baseNameOf(i), inodeHash); + } + done++; + act.progress(done, paths.size()); + } +} + +static string showBytes(unsigned long long bytes) +{ + return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str(); +} + +void LocalStore::optimiseStore() +{ + OptimiseStats stats; + + optimiseStore(stats); + + printInfo( + format("%1% freed by hard-linking %2% files") + % showBytes(stats.bytesFreed) + % stats.filesLinked); +} + +void LocalStore::optimisePath(const Path & path) +{ + OptimiseStats stats; + InodeHash inodeHash; + + if (settings.autoOptimiseStore) optimisePath_(nullptr, stats, path, inodeHash); +} + + +} diff --git a/third_party/nix/src/libstore/parsed-derivations.cc b/third_party/nix/src/libstore/parsed-derivations.cc new file mode 100644 index 000000000..87be8a24e --- /dev/null +++ b/third_party/nix/src/libstore/parsed-derivations.cc @@ -0,0 +1,116 @@ +#include "parsed-derivations.hh" + +namespace nix { + +ParsedDerivation::ParsedDerivation(const Path & drvPath, BasicDerivation & drv) + : drvPath(drvPath), drv(drv) +{ + /* Parse the __json attribute, if any. */ + auto jsonAttr = drv.env.find("__json"); + if (jsonAttr != drv.env.end()) { + try { + structuredAttrs = nlohmann::json::parse(jsonAttr->second); + } catch (std::exception & e) { + throw Error("cannot process __json attribute of '%s': %s", drvPath, e.what()); + } + } +} + +std::optional ParsedDerivation::getStringAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a string", name, drvPath); + return i->get(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return i->second; + } +} + +bool ParsedDerivation::getBoolAttr(const std::string & name, bool def) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return def; + else { + if (!i->is_boolean()) + throw Error("attribute '%s' of derivation '%s' must be a Boolean", name, drvPath); + return i->get(); + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return def; + else + return i->second == "1"; + } +} + +std::optional ParsedDerivation::getStringsAttr(const std::string & name) const +{ + if (structuredAttrs) { + auto i = structuredAttrs->find(name); + if (i == structuredAttrs->end()) + return {}; + else { + if (!i->is_array()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + Strings res; + for (auto j = i->begin(); j != i->end(); ++j) { + if (!j->is_string()) + throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, drvPath); + res.push_back(j->get()); + } + return res; + } + } else { + auto i = drv.env.find(name); + if (i == drv.env.end()) + return {}; + else + return tokenizeString(i->second); + } +} + +StringSet ParsedDerivation::getRequiredSystemFeatures() const +{ + StringSet res; + for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) + res.insert(i); + return res; +} + +bool ParsedDerivation::canBuildLocally() const +{ + if (drv.platform != settings.thisSystem.get() + && !settings.extraPlatforms.get().count(drv.platform) + && !drv.isBuiltin()) + return false; + + for (auto & feature : getRequiredSystemFeatures()) + if (!settings.systemFeatures.get().count(feature)) return false; + + return true; +} + +bool ParsedDerivation::willBuildLocally() const +{ + return getBoolAttr("preferLocalBuild") && canBuildLocally(); +} + +bool ParsedDerivation::substitutesAllowed() const +{ + return getBoolAttr("allowSubstitutes", true); +} + +} diff --git a/third_party/nix/src/libstore/parsed-derivations.hh b/third_party/nix/src/libstore/parsed-derivations.hh new file mode 100644 index 000000000..9bde4b4dc --- /dev/null +++ b/third_party/nix/src/libstore/parsed-derivations.hh @@ -0,0 +1,37 @@ +#include "derivations.hh" + +#include + +namespace nix { + +class ParsedDerivation +{ + Path drvPath; + BasicDerivation & drv; + std::optional structuredAttrs; + +public: + + ParsedDerivation(const Path & drvPath, BasicDerivation & drv); + + const std::optional & getStructuredAttrs() const + { + return structuredAttrs; + } + + std::optional getStringAttr(const std::string & name) const; + + bool getBoolAttr(const std::string & name, bool def = false) const; + + std::optional getStringsAttr(const std::string & name) const; + + StringSet getRequiredSystemFeatures() const; + + bool canBuildLocally() const; + + bool willBuildLocally() const; + + bool substitutesAllowed() const; +}; + +} diff --git a/third_party/nix/src/libstore/pathlocks.cc b/third_party/nix/src/libstore/pathlocks.cc new file mode 100644 index 000000000..2635e3940 --- /dev/null +++ b/third_party/nix/src/libstore/pathlocks.cc @@ -0,0 +1,178 @@ +#include "pathlocks.hh" +#include "util.hh" +#include "sync.hh" + +#include +#include + +#include +#include +#include +#include + + +namespace nix { + + +AutoCloseFD openLockFile(const Path & path, bool create) +{ + AutoCloseFD fd; + + fd = open(path.c_str(), O_CLOEXEC | O_RDWR | (create ? O_CREAT : 0), 0600); + if (!fd && (create || errno != ENOENT)) + throw SysError(format("opening lock file '%1%'") % path); + + return fd; +} + + +void deleteLockFile(const Path & path, int fd) +{ + /* Get rid of the lock file. Have to be careful not to introduce + races. Write a (meaningless) token to the file to indicate to + other processes waiting on this lock that the lock is stale + (deleted). */ + unlink(path.c_str()); + writeFull(fd, "d"); + /* Note that the result of unlink() is ignored; removing the lock + file is an optimisation, not a necessity. */ +} + + +bool lockFile(int fd, LockType lockType, bool wait) +{ + int type; + if (lockType == ltRead) type = LOCK_SH; + else if (lockType == ltWrite) type = LOCK_EX; + else if (lockType == ltNone) type = LOCK_UN; + else abort(); + + if (wait) { + while (flock(fd, type) != 0) { + checkInterrupt(); + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + else + return false; + } + } else { + while (flock(fd, type | LOCK_NB) != 0) { + checkInterrupt(); + if (errno == EWOULDBLOCK) return false; + if (errno != EINTR) + throw SysError(format("acquiring/releasing lock")); + } + } + + return true; +} + + +PathLocks::PathLocks() + : deletePaths(false) +{ +} + + +PathLocks::PathLocks(const PathSet & paths, const string & waitMsg) + : deletePaths(false) +{ + lockPaths(paths, waitMsg); +} + + +bool PathLocks::lockPaths(const PathSet & paths, + const string & waitMsg, bool wait) +{ + assert(fds.empty()); + + /* Note that `fds' is built incrementally so that the destructor + will only release those locks that we have already acquired. */ + + /* Acquire the lock for each path in sorted order. This ensures + that locks are always acquired in the same order, thus + preventing deadlocks. */ + for (auto & path : paths) { + checkInterrupt(); + Path lockPath = path + ".lock"; + + debug(format("locking path '%1%'") % path); + + AutoCloseFD fd; + + while (1) { + + /* Open/create the lock file. */ + fd = openLockFile(lockPath, true); + + /* Acquire an exclusive lock. */ + if (!lockFile(fd.get(), ltWrite, false)) { + if (wait) { + if (waitMsg != "") printError(waitMsg); + lockFile(fd.get(), ltWrite, true); + } else { + /* Failed to lock this path; release all other + locks. */ + unlock(); + return false; + } + } + + debug(format("lock acquired on '%1%'") % lockPath); + + /* Check that the lock file hasn't become stale (i.e., + hasn't been unlinked). */ + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError(format("statting lock file '%1%'") % lockPath); + if (st.st_size != 0) + /* This lock file has been unlinked, so we're holding + a lock on a deleted file. This means that other + processes may create and acquire a lock on + `lockPath', and proceed. So we must retry. */ + debug(format("open lock file '%1%' has become stale") % lockPath); + else + break; + } + + /* Use borrow so that the descriptor isn't closed. */ + fds.push_back(FDPair(fd.release(), lockPath)); + } + + return true; +} + + +PathLocks::~PathLocks() +{ + try { + unlock(); + } catch (...) { + ignoreException(); + } +} + + +void PathLocks::unlock() +{ + for (auto & i : fds) { + if (deletePaths) deleteLockFile(i.second, i.first); + + if (close(i.first) == -1) + printError( + format("error (ignored): cannot close lock file on '%1%'") % i.second); + + debug(format("lock released on '%1%'") % i.second); + } + + fds.clear(); +} + + +void PathLocks::setDeletion(bool deletePaths) +{ + this->deletePaths = deletePaths; +} + + +} diff --git a/third_party/nix/src/libstore/pathlocks.hh b/third_party/nix/src/libstore/pathlocks.hh new file mode 100644 index 000000000..411da0222 --- /dev/null +++ b/third_party/nix/src/libstore/pathlocks.hh @@ -0,0 +1,38 @@ +#pragma once + +#include "util.hh" + +namespace nix { + +/* Open (possibly create) a lock file and return the file descriptor. + -1 is returned if create is false and the lock could not be opened + because it doesn't exist. Any other error throws an exception. */ +AutoCloseFD openLockFile(const Path & path, bool create); + +/* Delete an open lock file. */ +void deleteLockFile(const Path & path, int fd); + +enum LockType { ltRead, ltWrite, ltNone }; + +bool lockFile(int fd, LockType lockType, bool wait); + +class PathLocks +{ +private: + typedef std::pair FDPair; + list fds; + bool deletePaths; + +public: + PathLocks(); + PathLocks(const PathSet & paths, + const string & waitMsg = ""); + bool lockPaths(const PathSet & _paths, + const string & waitMsg = "", + bool wait = true); + ~PathLocks(); + void unlock(); + void setDeletion(bool deletePaths); +}; + +} diff --git a/third_party/nix/src/libstore/profiles.cc b/third_party/nix/src/libstore/profiles.cc new file mode 100644 index 000000000..4c6af567a --- /dev/null +++ b/third_party/nix/src/libstore/profiles.cc @@ -0,0 +1,259 @@ +#include "profiles.hh" +#include "store-api.hh" +#include "util.hh" + +#include +#include +#include +#include +#include + + +namespace nix { + + +static bool cmpGensByNumber(const Generation & a, const Generation & b) +{ + return a.number < b.number; +} + + +/* Parse a generation name of the format + `--link'. */ +static int parseName(const string & profileName, const string & name) +{ + if (string(name, 0, profileName.size() + 1) != profileName + "-") return -1; + string s = string(name, profileName.size() + 1); + string::size_type p = s.find("-link"); + if (p == string::npos) return -1; + int n; + if (string2Int(string(s, 0, p), n) && n >= 0) + return n; + else + return -1; +} + + + +Generations findGenerations(Path profile, int & curGen) +{ + Generations gens; + + Path profileDir = dirOf(profile); + string profileName = baseNameOf(profile); + + for (auto & i : readDirectory(profileDir)) { + int n; + if ((n = parseName(profileName, i.name)) != -1) { + Generation gen; + gen.path = profileDir + "/" + i.name; + gen.number = n; + struct stat st; + if (lstat(gen.path.c_str(), &st) != 0) + throw SysError(format("statting '%1%'") % gen.path); + gen.creationTime = st.st_mtime; + gens.push_back(gen); + } + } + + gens.sort(cmpGensByNumber); + + curGen = pathExists(profile) + ? parseName(profileName, readLink(profile)) + : -1; + + return gens; +} + + +static void makeName(const Path & profile, unsigned int num, + Path & outLink) +{ + Path prefix = (format("%1%-%2%") % profile % num).str(); + outLink = prefix + "-link"; +} + + +Path createGeneration(ref store, Path profile, Path outPath) +{ + /* The new generation number should be higher than old the + previous ones. */ + int dummy; + Generations gens = findGenerations(profile, dummy); + + unsigned int num; + if (gens.size() > 0) { + Generation last = gens.back(); + + if (readLink(last.path) == outPath) { + /* We only create a new generation symlink if it differs + from the last one. + + This helps keeping gratuitous installs/rebuilds from piling + up uncontrolled numbers of generations, cluttering up the + UI like grub. */ + return last.path; + } + + num = gens.back().number; + } else { + num = 0; + } + + /* Create the new generation. Note that addPermRoot() blocks if + the garbage collector is running to prevent the stuff we've + built from moving from the temporary roots (which the GC knows) + to the permanent roots (of which the GC would have a stale + view). If we didn't do it this way, the GC might remove the + user environment etc. we've just built. */ + Path generation; + makeName(profile, num + 1, generation); + store->addPermRoot(outPath, generation, false, true); + + return generation; +} + + +static void removeFile(const Path & path) +{ + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink '%1%'") % path); +} + + +void deleteGeneration(const Path & profile, unsigned int gen) +{ + Path generation; + makeName(profile, gen, generation); + removeFile(generation); +} + + +static void deleteGeneration2(const Path & profile, unsigned int gen, bool dryRun) +{ + if (dryRun) + printInfo(format("would remove generation %1%") % gen); + else { + printInfo(format("removing generation %1%") % gen); + deleteGeneration(profile, gen); + } +} + + +void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + Generations gens = findGenerations(profile, curGen); + + if (gensToDelete.find(curGen) != gensToDelete.end()) + throw Error(format("cannot delete current generation of profile %1%'") % profile); + + for (auto & i : gens) { + if (gensToDelete.find(i.number) == gensToDelete.end()) continue; + deleteGeneration2(profile, i.number, dryRun); + } +} + +void deleteGenerationsGreaterThan(const Path & profile, int max, bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + bool fromCurGen = false; + Generations gens = findGenerations(profile, curGen); + for (auto i = gens.rbegin(); i != gens.rend(); ++i) { + if (i->number == curGen) { + fromCurGen = true; + max--; + continue; + } + if (fromCurGen) { + if (max) { + max--; + continue; + } + deleteGeneration2(profile, i->number, dryRun); + } + } +} + +void deleteOldGenerations(const Path & profile, bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + Generations gens = findGenerations(profile, curGen); + + for (auto & i : gens) + if (i.number != curGen) + deleteGeneration2(profile, i.number, dryRun); +} + + +void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun) +{ + PathLocks lock; + lockProfile(lock, profile); + + int curGen; + Generations gens = findGenerations(profile, curGen); + + bool canDelete = false; + for (auto i = gens.rbegin(); i != gens.rend(); ++i) + if (canDelete) { + assert(i->creationTime < t); + if (i->number != curGen) + deleteGeneration2(profile, i->number, dryRun); + } else if (i->creationTime < t) { + /* We may now start deleting generations, but we don't + delete this generation yet, because this generation was + still the one that was active at the requested point in + time. */ + canDelete = true; + } +} + + +void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun) +{ + time_t curTime = time(0); + string strDays = string(timeSpec, 0, timeSpec.size() - 1); + int days; + + if (!string2Int(strDays, days) || days < 1) + throw Error(format("invalid number of days specifier '%1%'") % timeSpec); + + time_t oldTime = curTime - days * 24 * 3600; + + deleteGenerationsOlderThan(profile, oldTime, dryRun); +} + + +void switchLink(Path link, Path target) +{ + /* Hacky. */ + if (dirOf(target) == dirOf(link)) target = baseNameOf(target); + + replaceSymlink(target, link); +} + + +void lockProfile(PathLocks & lock, const Path & profile) +{ + lock.lockPaths({profile}, (format("waiting for lock on profile '%1%'") % profile).str()); + lock.setDeletion(true); +} + + +string optimisticLockProfile(const Path & profile) +{ + return pathExists(profile) ? readLink(profile) : ""; +} + + +} diff --git a/third_party/nix/src/libstore/profiles.hh b/third_party/nix/src/libstore/profiles.hh new file mode 100644 index 000000000..5fa1533de --- /dev/null +++ b/third_party/nix/src/libstore/profiles.hh @@ -0,0 +1,67 @@ +#pragma once + +#include "types.hh" +#include "pathlocks.hh" + +#include + + +namespace nix { + + +struct Generation +{ + int number; + Path path; + time_t creationTime; + Generation() + { + number = -1; + } + operator bool() const + { + return number != -1; + } +}; + +typedef list Generations; + + +/* Returns the list of currently present generations for the specified + profile, sorted by generation number. */ +Generations findGenerations(Path profile, int & curGen); + +class LocalFSStore; + +Path createGeneration(ref store, Path profile, Path outPath); + +void deleteGeneration(const Path & profile, unsigned int gen); + +void deleteGenerations(const Path & profile, const std::set & gensToDelete, bool dryRun); + +void deleteGenerationsGreaterThan(const Path & profile, const int max, bool dryRun); + +void deleteOldGenerations(const Path & profile, bool dryRun); + +void deleteGenerationsOlderThan(const Path & profile, time_t t, bool dryRun); + +void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, bool dryRun); + +void switchLink(Path link, Path target); + +/* Ensure exclusive access to a profile. Any command that modifies + the profile first acquires this lock. */ +void lockProfile(PathLocks & lock, const Path & profile); + +/* Optimistic locking is used by long-running operations like `nix-env + -i'. Instead of acquiring the exclusive lock for the entire + duration of the operation, we just perform the operation + optimistically (without an exclusive lock), and check at the end + whether the profile changed while we were busy (i.e., the symlink + target changed). If so, the operation is restarted. Restarting is + generally cheap, since the build results are still in the Nix + store. Most of the time, only the user environment has to be + rebuilt. */ +string optimisticLockProfile(const Path & profile); + +} diff --git a/third_party/nix/src/libstore/references.cc b/third_party/nix/src/libstore/references.cc new file mode 100644 index 000000000..5b7eb1f84 --- /dev/null +++ b/third_party/nix/src/libstore/references.cc @@ -0,0 +1,122 @@ +#include "references.hh" +#include "hash.hh" +#include "util.hh" +#include "archive.hh" + +#include +#include + + +namespace nix { + + +static unsigned int refLength = 32; /* characters */ + + +static void search(const unsigned char * s, size_t len, + StringSet & hashes, StringSet & seen) +{ + static bool initialised = false; + static bool isBase32[256]; + if (!initialised) { + for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; + for (unsigned int i = 0; i < base32Chars.size(); ++i) + isBase32[(unsigned char) base32Chars[i]] = true; + initialised = true; + } + + for (size_t i = 0; i + refLength <= len; ) { + int j; + bool match = true; + for (j = refLength - 1; j >= 0; --j) + if (!isBase32[(unsigned char) s[i + j]]) { + i += j + 1; + match = false; + break; + } + if (!match) continue; + string ref((const char *) s + i, refLength); + if (hashes.find(ref) != hashes.end()) { + debug(format("found reference to '%1%' at offset '%2%'") + % ref % i); + seen.insert(ref); + hashes.erase(ref); + } + ++i; + } +} + + +struct RefScanSink : Sink +{ + HashSink hashSink; + StringSet hashes; + StringSet seen; + + string tail; + + RefScanSink() : hashSink(htSHA256) { } + + void operator () (const unsigned char * data, size_t len); +}; + + +void RefScanSink::operator () (const unsigned char * data, size_t len) +{ + hashSink(data, len); + + /* It's possible that a reference spans the previous and current + fragment, so search in the concatenation of the tail of the + previous fragment and the start of the current fragment. */ + string s = tail + string((const char *) data, len > refLength ? refLength : len); + search((const unsigned char *) s.data(), s.size(), hashes, seen); + + search(data, len, hashes, seen); + + size_t tailLen = len <= refLength ? len : refLength; + tail = + string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) + + string((const char *) data + len - tailLen, tailLen); +} + + +PathSet scanForReferences(const string & path, + const PathSet & refs, HashResult & hash) +{ + RefScanSink sink; + std::map backMap; + + /* For efficiency (and a higher hit rate), just search for the + hash part of the file name. (This assumes that all references + have the form `HASH-bla'). */ + for (auto & i : refs) { + string baseName = baseNameOf(i); + string::size_type pos = baseName.find('-'); + if (pos == string::npos) + throw Error(format("bad reference '%1%'") % i); + string s = string(baseName, 0, pos); + assert(s.size() == refLength); + assert(backMap.find(s) == backMap.end()); + // parseHash(htSHA256, s); + sink.hashes.insert(s); + backMap[s] = i; + } + + /* Look for the hashes in the NAR dump of the path. */ + dumpPath(path, sink); + + /* Map the hashes found back to their store paths. */ + PathSet found; + for (auto & i : sink.seen) { + std::map::iterator j; + if ((j = backMap.find(i)) == backMap.end()) abort(); + found.insert(j->second); + } + + hash = sink.hashSink.finish(); + + return found; +} + + +} diff --git a/third_party/nix/src/libstore/references.hh b/third_party/nix/src/libstore/references.hh new file mode 100644 index 000000000..013809d12 --- /dev/null +++ b/third_party/nix/src/libstore/references.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +namespace nix { + +PathSet scanForReferences(const Path & path, const PathSet & refs, + HashResult & hash); + +} diff --git a/third_party/nix/src/libstore/remote-fs-accessor.cc b/third_party/nix/src/libstore/remote-fs-accessor.cc new file mode 100644 index 000000000..5233fb2c2 --- /dev/null +++ b/third_party/nix/src/libstore/remote-fs-accessor.cc @@ -0,0 +1,129 @@ +#include "remote-fs-accessor.hh" +#include "nar-accessor.hh" +#include "json.hh" + +#include +#include +#include + +namespace nix { + +RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) + : store(store) + , cacheDir(cacheDir) +{ + if (cacheDir != "") + createDirs(cacheDir); +} + +Path RemoteFSAccessor::makeCacheFile(const Path & storePath, const std::string & ext) +{ + assert(cacheDir != ""); + return fmt("%s/%s.%s", cacheDir, storePathToHash(storePath), ext); +} + +void RemoteFSAccessor::addToCache(const Path & storePath, const std::string & nar, + ref narAccessor) +{ + nars.emplace(storePath, narAccessor); + + if (cacheDir != "") { + try { + std::ostringstream str; + JSONPlaceholder jsonRoot(str); + listNar(jsonRoot, narAccessor, "", true); + writeFile(makeCacheFile(storePath, "ls"), str.str()); + + /* FIXME: do this asynchronously. */ + writeFile(makeCacheFile(storePath, "nar"), nar); + + } catch (...) { + ignoreException(); + } + } +} + +std::pair, Path> RemoteFSAccessor::fetch(const Path & path_) +{ + auto path = canonPath(path_); + + auto storePath = store->toStorePath(path); + std::string restPath = std::string(path, storePath.size()); + + if (!store->isValidPath(storePath)) + throw InvalidPath(format("path '%1%' is not a valid store path") % storePath); + + auto i = nars.find(storePath); + if (i != nars.end()) return {i->second, restPath}; + + StringSink sink; + std::string listing; + Path cacheFile; + + if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath, "nar"))) { + + try { + listing = nix::readFile(makeCacheFile(storePath, "ls")); + + auto narAccessor = makeLazyNarAccessor(listing, + [cacheFile](uint64_t offset, uint64_t length) { + + AutoCloseFD fd = open(cacheFile.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening NAR cache file '%s'", cacheFile); + + if (lseek(fd.get(), offset, SEEK_SET) != (off_t) offset) + throw SysError("seeking in '%s'", cacheFile); + + std::string buf(length, 0); + readFull(fd.get(), (unsigned char *) buf.data(), length); + + return buf; + }); + + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; + + } catch (SysError &) { } + + try { + *sink.s = nix::readFile(cacheFile); + + auto narAccessor = makeNarAccessor(sink.s); + nars.emplace(storePath, narAccessor); + return {narAccessor, restPath}; + + } catch (SysError &) { } + } + + store->narFromPath(storePath, sink); + auto narAccessor = makeNarAccessor(sink.s); + addToCache(storePath, *sink.s, narAccessor); + return {narAccessor, restPath}; +} + +FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +{ + auto res = fetch(path); + return res.first->stat(res.second); +} + +StringSet RemoteFSAccessor::readDirectory(const Path & path) +{ + auto res = fetch(path); + return res.first->readDirectory(res.second); +} + +std::string RemoteFSAccessor::readFile(const Path & path) +{ + auto res = fetch(path); + return res.first->readFile(res.second); +} + +std::string RemoteFSAccessor::readLink(const Path & path) +{ + auto res = fetch(path); + return res.first->readLink(res.second); +} + +} diff --git a/third_party/nix/src/libstore/remote-fs-accessor.hh b/third_party/nix/src/libstore/remote-fs-accessor.hh new file mode 100644 index 000000000..4afb3be95 --- /dev/null +++ b/third_party/nix/src/libstore/remote-fs-accessor.hh @@ -0,0 +1,40 @@ +#pragma once + +#include "fs-accessor.hh" +#include "ref.hh" +#include "store-api.hh" + +namespace nix { + +class RemoteFSAccessor : public FSAccessor +{ + ref store; + + std::map> nars; + + Path cacheDir; + + std::pair, Path> fetch(const Path & path_); + + friend class BinaryCacheStore; + + Path makeCacheFile(const Path & storePath, const std::string & ext); + + void addToCache(const Path & storePath, const std::string & nar, + ref narAccessor); + +public: + + RemoteFSAccessor(ref store, + const /* FIXME: use std::optional */ Path & cacheDir = ""); + + Stat stat(const Path & path) override; + + StringSet readDirectory(const Path & path) override; + + std::string readFile(const Path & path) override; + + std::string readLink(const Path & path) override; +}; + +} diff --git a/third_party/nix/src/libstore/remote-store.cc b/third_party/nix/src/libstore/remote-store.cc new file mode 100644 index 000000000..f34369d8f --- /dev/null +++ b/third_party/nix/src/libstore/remote-store.cc @@ -0,0 +1,817 @@ +#include "serialise.hh" +#include "util.hh" +#include "remote-store.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" +#include "derivations.hh" +#include "pool.hh" +#include "finally.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace nix { + + +Path readStorePath(Store & store, Source & from) +{ + Path path = readString(from); + store.assertStorePath(path); + return path; +} + + +template T readStorePaths(Store & store, Source & from) +{ + T paths = readStrings(from); + for (auto & i : paths) store.assertStorePath(i); + return paths; +} + +template PathSet readStorePaths(Store & store, Source & from); +template Paths readStorePaths(Store & store, Source & from); + +/* TODO: Separate these store impls into different files, give them better names */ +RemoteStore::RemoteStore(const Params & params) + : Store(params) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnectionWrapper(); }, + [this](const ref & r) { + return + r->to.good() + && r->from.good() + && std::chrono::duration_cast( + std::chrono::steady_clock::now() - r->startTime).count() < maxConnectionAge; + } + )) +{ +} + + +ref RemoteStore::openConnectionWrapper() +{ + if (failed) + throw Error("opening a connection to remote store '%s' previously failed", getUri()); + try { + return openConnection(); + } catch (...) { + failed = true; + throw; + } +} + + +UDSRemoteStore::UDSRemoteStore(const Params & params) + : Store(params) + , LocalFSStore(params) + , RemoteStore(params) +{ +} + + +UDSRemoteStore::UDSRemoteStore(std::string socket_path, const Params & params) + : Store(params) + , LocalFSStore(params) + , RemoteStore(params) + , path(socket_path) +{ +} + + +std::string UDSRemoteStore::getUri() +{ + if (path) { + return std::string("unix://") + *path; + } else { + return "daemon"; + } +} + + +ref UDSRemoteStore::openConnection() +{ + auto conn = make_ref(); + + /* Connect to a daemon that does the privileged work for us. */ + conn->fd = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!conn->fd) + throw SysError("cannot create Unix domain socket"); + closeOnExec(conn->fd.get()); + + string socketPath = path ? *path : settings.nixDaemonSocketFile; + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPath.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket path '%1%' is too long") % socketPath); + strcpy(addr.sun_path, socketPath.c_str()); + + if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at '%1%'") % socketPath); + + conn->from.fd = conn->fd.get(); + conn->to.fd = conn->fd.get(); + + conn->startTime = std::chrono::steady_clock::now(); + + initConnection(*conn); + + return conn; +} + + +void RemoteStore::initConnection(Connection & conn) +{ + /* Send the magic greeting, check for the reply. */ + try { + conn.to << WORKER_MAGIC_1; + conn.to.flush(); + unsigned int magic = readInt(conn.from); + if (magic != WORKER_MAGIC_2) throw Error("protocol mismatch"); + + conn.from >> conn.daemonVersion; + if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) + throw Error("the Nix daemon version is too old"); + conn.to << PROTOCOL_VERSION; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { + int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; + if (cpu != -1) + conn.to << 1 << cpu; + else + conn.to << 0; + } + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) + conn.to << false; + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); + } + catch (Error & e) { + throw Error("cannot open connection to remote store '%s': %s", getUri(), e.what()); + } + + setOptions(conn); +} + + +void RemoteStore::setOptions(Connection & conn) +{ + conn.to << wopSetOptions + << settings.keepFailed + << settings.keepGoing + << settings.tryFallback + << verbosity + << settings.maxBuildJobs + << settings.maxSilentTime + << true + << (settings.verboseBuild ? lvlError : lvlVomit) + << 0 // obsolete log type + << 0 /* obsolete print build trace */ + << settings.buildCores + << settings.useSubstitutes; + + if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + std::map overrides; + globalConfig.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(settings.showTrace.name); + conn.to << overrides.size(); + for (auto & i : overrides) + conn.to << i.first << i.second.value; + } + + auto ex = conn.processStderr(); + if (ex) std::rethrow_exception(ex); +} + + +/* A wrapper around Pool::Handle that marks + the connection as bad (causing it to be closed) if a non-daemon + exception is thrown before the handle is closed. Such an exception + causes a deviation from the expected protocol and therefore a + desynchronization between the client and daemon. */ +struct ConnectionHandle +{ + Pool::Handle handle; + bool daemonException = false; + + ConnectionHandle(Pool::Handle && handle) + : handle(std::move(handle)) + { } + + ConnectionHandle(ConnectionHandle && h) + : handle(std::move(h.handle)) + { } + + ~ConnectionHandle() + { + if (!daemonException && std::uncaught_exception()) { + handle.markBad(); + debug("closing daemon connection because of an exception"); + } + } + + RemoteStore::Connection * operator -> () { return &*handle; } + + void processStderr(Sink * sink = 0, Source * source = 0) + { + auto ex = handle->processStderr(sink, source); + if (ex) { + daemonException = true; + std::rethrow_exception(ex); + } + } +}; + + +ConnectionHandle RemoteStore::getConnection() +{ + return ConnectionHandle(connections->get()); +} + + +bool RemoteStore::isValidPathUncached(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopIsValidPath << path; + conn.processStderr(); + return readInt(conn->from); +} + + +PathSet RemoteStore::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + PathSet res; + for (auto & i : paths) + if (isValidPath(i)) res.insert(i); + return res; + } else { + conn->to << wopQueryValidPaths << paths; + conn.processStderr(); + return readStorePaths(*this, conn->from); + } +} + + +PathSet RemoteStore::queryAllValidPaths() +{ + auto conn(getConnection()); + conn->to << wopQueryAllValidPaths; + conn.processStderr(); + return readStorePaths(*this, conn->from); +} + + +PathSet RemoteStore::querySubstitutablePaths(const PathSet & paths) +{ + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + PathSet res; + for (auto & i : paths) { + conn->to << wopHasSubstitutes << i; + conn.processStderr(); + if (readInt(conn->from)) res.insert(i); + } + return res; + } else { + conn->to << wopQuerySubstitutablePaths << paths; + conn.processStderr(); + return readStorePaths(*this, conn->from); + } +} + + +void RemoteStore::querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) +{ + if (paths.empty()) return; + + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + + for (auto & i : paths) { + SubstitutablePathInfo info; + conn->to << wopQuerySubstitutablePathInfo << i; + conn.processStderr(); + unsigned int reply = readInt(conn->from); + if (reply == 0) continue; + info.deriver = readString(conn->from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths(*this, conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + infos[i] = info; + } + + } else { + + conn->to << wopQuerySubstitutablePathInfos << paths; + conn.processStderr(); + size_t count = readNum(conn->from); + for (size_t n = 0; n < count; n++) { + Path path = readStorePath(*this, conn->from); + SubstitutablePathInfo & info(infos[path]); + info.deriver = readString(conn->from); + if (info.deriver != "") assertStorePath(info.deriver); + info.references = readStorePaths(*this, conn->from); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); + } + + } +} + + +void RemoteStore::queryPathInfoUncached(const Path & path, + Callback> callback) noexcept +{ + try { + std::shared_ptr info; + { + auto conn(getConnection()); + conn->to << wopQueryPathInfo << path; + try { + conn.processStderr(); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(e.what()); + throw; + } + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { + bool valid; conn->from >> valid; + if (!valid) throw InvalidPath(format("path '%s' is not valid") % path); + } + info = std::make_shared(); + info->path = path; + info->deriver = readString(conn->from); + if (info->deriver != "") assertStorePath(info->deriver); + info->narHash = Hash(readString(conn->from), htSHA256); + info->references = readStorePaths(*this, conn->from); + conn->from >> info->registrationTime >> info->narSize; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) { + conn->from >> info->ultimate; + info->sigs = readStrings(conn->from); + conn->from >> info->ca; + } + } + callback(std::move(info)); + } catch (...) { callback.rethrow(); } +} + + +void RemoteStore::queryReferrers(const Path & path, + PathSet & referrers) +{ + auto conn(getConnection()); + conn->to << wopQueryReferrers << path; + conn.processStderr(); + PathSet referrers2 = readStorePaths(*this, conn->from); + referrers.insert(referrers2.begin(), referrers2.end()); +} + + +PathSet RemoteStore::queryValidDerivers(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopQueryValidDerivers << path; + conn.processStderr(); + return readStorePaths(*this, conn->from); +} + + +PathSet RemoteStore::queryDerivationOutputs(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputs << path; + conn.processStderr(); + return readStorePaths(*this, conn->from); +} + + +PathSet RemoteStore::queryDerivationOutputNames(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopQueryDerivationOutputNames << path; + conn.processStderr(); + return readStrings(conn->from); +} + + +Path RemoteStore::queryPathFromHashPart(const string & hashPart) +{ + auto conn(getConnection()); + conn->to << wopQueryPathFromHashPart << hashPart; + conn.processStderr(); + Path path = readString(conn->from); + if (!path.empty()) assertStorePath(path); + return path; +} + + +void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr accessor) +{ + auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { + conn->to << wopImportPaths; + + auto source2 = sinkToSource([&](Sink & sink) { + sink << 1 // == path follows + ; + copyNAR(source, sink); + sink + << exportMagic + << info.path + << info.references + << info.deriver + << 0 // == no legacy signature + << 0 // == no path follows + ; + }); + + conn.processStderr(0, source2.get()); + + auto importedPaths = readStorePaths(*this, conn->from); + assert(importedPaths.size() <= 1); + } + + else { + conn->to << wopAddToStoreNar + << info.path << info.deriver << info.narHash.to_string(Base16, false) + << info.references << info.registrationTime << info.narSize + << info.ultimate << info.sigs << info.ca + << repair << !checkSigs; + bool tunnel = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21; + if (!tunnel) copyNAR(source, conn->to); + conn.processStderr(0, tunnel ? &source : nullptr); + } +} + + +Path RemoteStore::addToStore(const string & name, const Path & _srcPath, + bool recursive, HashType hashAlgo, PathFilter & filter, RepairFlag repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + + auto conn(getConnection()); + + Path srcPath(absPath(_srcPath)); + + conn->to << wopAddToStore << name + << ((hashAlgo == htSHA256 && recursive) ? 0 : 1) /* backwards compatibility hack */ + << (recursive ? 1 : 0) + << printHashType(hashAlgo); + + try { + conn->to.written = 0; + conn->to.warn = true; + connections->incCapacity(); + { + Finally cleanup([&]() { connections->decCapacity(); }); + dumpPath(srcPath, conn->to, filter); + } + conn->to.warn = false; + conn.processStderr(); + } catch (SysError & e) { + /* Daemon closed while we were sending the path. Probably OOM + or I/O error. */ + if (e.errNo == EPIPE) + try { + conn.processStderr(); + } catch (EndOfFile & e) { } + throw; + } + + return readStorePath(*this, conn->from); +} + + +Path RemoteStore::addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) +{ + if (repair) throw Error("repairing is not supported when building through the Nix daemon"); + + auto conn(getConnection()); + conn->to << wopAddTextToStore << name << s << references; + + conn.processStderr(); + return readStorePath(*this, conn->from); +} + + +void RemoteStore::buildPaths(const PathSet & drvPaths, BuildMode buildMode) +{ + auto conn(getConnection()); + conn->to << wopBuildPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13) { + conn->to << drvPaths; + if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + conn->to << buildMode; + else + /* Old daemons did not take a 'buildMode' parameter, so we + need to validate it here on the client side. */ + if (buildMode != bmNormal) + throw Error("repairing or checking is not supported when building through the Nix daemon"); + } else { + /* For backwards compatibility with old daemons, strip output + identifiers. */ + PathSet drvPaths2; + for (auto & i : drvPaths) + drvPaths2.insert(string(i, 0, i.find('!'))); + conn->to << drvPaths2; + } + conn.processStderr(); + readInt(conn->from); +} + + +BuildResult RemoteStore::buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(getConnection()); + conn->to << wopBuildDerivation << drvPath << drv << buildMode; + conn.processStderr(); + BuildResult res; + unsigned int status; + conn->from >> status >> res.errorMsg; + res.status = (BuildResult::Status) status; + return res; +} + + +void RemoteStore::ensurePath(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopEnsurePath << path; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::addTempRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopAddTempRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::addIndirectRoot(const Path & path) +{ + auto conn(getConnection()); + conn->to << wopAddIndirectRoot << path; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::syncWithGC() +{ + auto conn(getConnection()); + conn->to << wopSyncWithGC; + conn.processStderr(); + readInt(conn->from); +} + + +Roots RemoteStore::findRoots(bool censor) +{ + auto conn(getConnection()); + conn->to << wopFindRoots; + conn.processStderr(); + size_t count = readNum(conn->from); + Roots result; + while (count--) { + Path link = readString(conn->from); + Path target = readStorePath(*this, conn->from); + result[target].emplace(link); + } + return result; +} + + +void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + auto conn(getConnection()); + + conn->to + << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness + << options.maxFreed + /* removed options */ + << 0 << 0 << 0; + + conn.processStderr(); + + results.paths = readStrings(conn->from); + results.bytesFreed = readLongLong(conn->from); + readLongLong(conn->from); // obsolete + + { + auto state_(Store::state.lock()); + state_->pathInfoCache.clear(); + } +} + + +void RemoteStore::optimiseStore() +{ + auto conn(getConnection()); + conn->to << wopOptimiseStore; + conn.processStderr(); + readInt(conn->from); +} + + +bool RemoteStore::verifyStore(bool checkContents, RepairFlag repair) +{ + auto conn(getConnection()); + conn->to << wopVerifyStore << checkContents << repair; + conn.processStderr(); + return readInt(conn->from); +} + + +void RemoteStore::addSignatures(const Path & storePath, const StringSet & sigs) +{ + auto conn(getConnection()); + conn->to << wopAddSignatures << storePath << sigs; + conn.processStderr(); + readInt(conn->from); +} + + +void RemoteStore::queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) +{ + { + auto conn(getConnection()); + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + // Don't hold the connection handle in the fallback case + // to prevent a deadlock. + goto fallback; + conn->to << wopQueryMissing << targets; + conn.processStderr(); + willBuild = readStorePaths(*this, conn->from); + willSubstitute = readStorePaths(*this, conn->from); + unknown = readStorePaths(*this, conn->from); + conn->from >> downloadSize >> narSize; + return; + } + + fallback: + return Store::queryMissing(targets, willBuild, willSubstitute, + unknown, downloadSize, narSize); +} + + +void RemoteStore::connect() +{ + auto conn(getConnection()); +} + + +unsigned int RemoteStore::getProtocol() +{ + auto conn(connections->get()); + return conn->daemonVersion; +} + + +void RemoteStore::flushBadConnections() +{ + connections->flushBad(); +} + + +RemoteStore::Connection::~Connection() +{ + try { + to.flush(); + } catch (...) { + ignoreException(); + } +} + + +static Logger::Fields readFields(Source & from) +{ + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type)) readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int) type); + } + return fields; +} + + +std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source) +{ + to.flush(); + + while (true) { + + auto msg = readNum(from); + + if (msg == STDERR_WRITE) { + string s = readString(from); + if (!sink) throw Error("no sink"); + (*sink)(s); + } + + else if (msg == STDERR_READ) { + if (!source) throw Error("no source"); + size_t len = readNum(from); + auto buf = std::make_unique(len); + writeString(buf.get(), source->read(buf.get(), len), to); + to.flush(); + } + + else if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = readInt(from); + return std::make_exception_ptr(Error(status, error)); + } + + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum(from); + auto lvl = (Verbosity) readInt(from); + auto type = (ActivityType) readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum(from); + logger->startActivity(act, lvl, type, s, fields, parent); + } + + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum(from); + logger->stopActivity(act); + } + + else if (msg == STDERR_RESULT) { + auto act = readNum(from); + auto type = (ResultType) readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } + + else if (msg == STDERR_LAST) + break; + + else + throw Error("got unknown message type %x from Nix daemon", msg); + } + + return nullptr; +} + +static std::string uriScheme = "unix://"; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared(std::string(uri, uriScheme.size()), params); +}); + +} diff --git a/third_party/nix/src/libstore/remote-store.hh b/third_party/nix/src/libstore/remote-store.hh new file mode 100644 index 000000000..1f375dd71 --- /dev/null +++ b/third_party/nix/src/libstore/remote-store.hh @@ -0,0 +1,161 @@ +#pragma once + +#include +#include + +#include "store-api.hh" + + +namespace nix { + + +class Pipe; +class Pid; +struct FdSink; +struct FdSource; +template class Pool; +struct ConnectionHandle; + + +/* FIXME: RemoteStore is a misnomer - should be something like + DaemonStore. */ +class RemoteStore : public virtual Store +{ +public: + + const Setting maxConnections{(Store*) this, 1, + "max-connections", "maximum number of concurrent connections to the Nix daemon"}; + + const Setting maxConnectionAge{(Store*) this, std::numeric_limits::max(), + "max-connection-age", "number of seconds to reuse a connection"}; + + virtual bool sameMachine() = 0; + + RemoteStore(const Params & params); + + /* Implementations of abstract store API methods. */ + + bool isValidPathUncached(const Path & path) override; + + PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + PathSet queryAllValidPaths() override; + + void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept override; + + void queryReferrers(const Path & path, PathSet & referrers) override; + + PathSet queryValidDerivers(const Path & path) override; + + PathSet queryDerivationOutputs(const Path & path) override; + + StringSet queryDerivationOutputNames(const Path & path) override; + + Path queryPathFromHashPart(const string & hashPart) override; + + PathSet querySubstitutablePaths(const PathSet & paths) override; + + void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) override; + + void addToStore(const ValidPathInfo & info, Source & nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) override; + + Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override; + + Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair) override; + + void buildPaths(const PathSet & paths, BuildMode buildMode) override; + + BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void ensurePath(const Path & path) override; + + void addTempRoot(const Path & path) override; + + void addIndirectRoot(const Path & path) override; + + void syncWithGC() override; + + Roots findRoots(bool censor) override; + + void collectGarbage(const GCOptions & options, GCResults & results) override; + + void optimiseStore() override; + + bool verifyStore(bool checkContents, RepairFlag repair) override; + + void addSignatures(const Path & storePath, const StringSet & sigs) override; + + void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize) override; + + void connect() override; + + unsigned int getProtocol() override; + + void flushBadConnections(); + +protected: + + struct Connection + { + AutoCloseFD fd; + FdSink to; + FdSource from; + unsigned int daemonVersion; + std::chrono::time_point startTime; + + virtual ~Connection(); + + std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0); + }; + + ref openConnectionWrapper(); + + virtual ref openConnection() = 0; + + void initConnection(Connection & conn); + + ref> connections; + + virtual void setOptions(Connection & conn); + + ConnectionHandle getConnection(); + + friend struct ConnectionHandle; + +private: + + std::atomic_bool failed{false}; + +}; + +class UDSRemoteStore : public LocalFSStore, public RemoteStore +{ +public: + + UDSRemoteStore(const Params & params); + UDSRemoteStore(std::string path, const Params & params); + + std::string getUri() override; + + bool sameMachine() + { return true; } + +private: + + ref openConnection() override; + std::optional path; +}; + + +} diff --git a/third_party/nix/src/libstore/s3-binary-cache-store.cc b/third_party/nix/src/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000..cd547a964 --- /dev/null +++ b/third_party/nix/src/libstore/s3-binary-cache-store.cc @@ -0,0 +1,431 @@ +#if ENABLE_S3 + +#include "s3.hh" +#include "s3-binary-cache-store.hh" +#include "nar-info.hh" +#include "nar-info-disk-cache.hh" +#include "globals.hh" +#include "compression.hh" +#include "download.hh" +#include "istringstream_nocopy.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Aws::Transfer; + +namespace nix { + +struct S3Error : public Error +{ + Aws::S3::S3Errors err; + S3Error(Aws::S3::S3Errors err, const FormatOrString & fs) + : Error(fs), err(err) { }; +}; + +/* Helper: given an Outcome, return R in case of success, or + throw an exception in case of an error. */ +template +R && checkAws(const FormatOrString & fs, Aws::Utils::Outcome && outcome) +{ + if (!outcome.IsSuccess()) + throw S3Error( + outcome.GetError().GetErrorType(), + fs.s + ": " + outcome.GetError().GetMessage()); + return outcome.GetResultWithOwnership(); +} + +class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem +{ + using Aws::Utils::Logging::FormattedLogSystem::FormattedLogSystem; + + void ProcessFormattedStatement(Aws::String && statement) override + { + debug("AWS: %s", chomp(statement)); + } +}; + +static void initAWS() +{ + static std::once_flag flag; + std::call_once(flag, []() { + Aws::SDKOptions options; + + /* We install our own OpenSSL locking function (see + shared.cc), so don't let aws-sdk-cpp override it. */ + options.cryptoOptions.initAndCleanupOpenSSL = false; + + if (verbosity >= lvlDebug) { + options.loggingOptions.logLevel = + verbosity == lvlDebug + ? Aws::Utils::Logging::LogLevel::Debug + : Aws::Utils::Logging::LogLevel::Trace; + options.loggingOptions.logger_create_fn = [options]() { + return std::make_shared(options.loggingOptions.logLevel); + }; + } + + Aws::InitAPI(options); + }); +} + +S3Helper::S3Helper(const string & profile, const string & region, const string & scheme, const string & endpoint) + : config(makeConfig(region, scheme, endpoint)) + , client(make_ref( + profile == "" + ? std::dynamic_pointer_cast( + std::make_shared()) + : std::dynamic_pointer_cast( + std::make_shared(profile.c_str())), + *config, + // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 +#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 + false, +#else + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, +#endif + endpoint.empty())) +{ +} + +/* Log AWS retries. */ +class RetryStrategy : public Aws::Client::DefaultRetryStrategy +{ + bool ShouldRetry(const Aws::Client::AWSError& error, long attemptedRetries) const override + { + auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); + if (retry) + printError("AWS error '%s' (%s), will retry in %d ms", + error.GetExceptionName(), error.GetMessage(), CalculateDelayBeforeNextRetry(error, attemptedRetries)); + return retry; + } +}; + +ref S3Helper::makeConfig(const string & region, const string & scheme, const string & endpoint) +{ + initAWS(); + auto res = make_ref(); + res->region = region; + if (!scheme.empty()) { + res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); + } + if (!endpoint.empty()) { + res->endpointOverride = endpoint; + } + res->requestTimeoutMs = 600 * 1000; + res->connectTimeoutMs = 5 * 1000; + res->retryStrategy = std::make_shared(); + res->caFile = settings.caFile; + return res; +} + +S3Helper::DownloadResult S3Helper::getObject( + const std::string & bucketName, const std::string & key) +{ + debug("fetching 's3://%s/%s'...", bucketName, key); + + auto request = + Aws::S3::Model::GetObjectRequest() + .WithBucket(bucketName) + .WithKey(key); + + request.SetResponseStreamFactory([&]() { + return Aws::New("STRINGSTREAM"); + }); + + DownloadResult res; + + auto now1 = std::chrono::steady_clock::now(); + + try { + + auto result = checkAws(fmt("AWS error fetching '%s'", key), + client->GetObject(request)); + + res.data = decompress(result.GetContentEncoding(), + dynamic_cast(result.GetBody()).str()); + + } catch (S3Error & e) { + if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw; + } + + auto now2 = std::chrono::steady_clock::now(); + + res.durationMs = std::chrono::duration_cast(now2 - now1).count(); + + return res; +} + +struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore +{ + const Setting profile{this, "", "profile", "The name of the AWS configuration profile to use."}; + const Setting region{this, Aws::Region::US_EAST_1, "region", {"aws-region"}}; + const Setting scheme{this, "", "scheme", "The scheme to use for S3 requests, https by default."}; + const Setting endpoint{this, "", "endpoint", "An optional override of the endpoint to use when talking to S3."}; + const Setting narinfoCompression{this, "", "narinfo-compression", "compression method for .narinfo files"}; + const Setting lsCompression{this, "", "ls-compression", "compression method for .ls files"}; + const Setting logCompression{this, "", "log-compression", "compression method for log/* files"}; + const Setting multipartUpload{ + this, false, "multipart-upload", "whether to use multi-part uploads"}; + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "size (in bytes) of each part in multi-part uploads"}; + + std::string bucketName; + + Stats stats; + + S3Helper s3Helper; + + S3BinaryCacheStoreImpl( + const Params & params, const std::string & bucketName) + : S3BinaryCacheStore(params) + , bucketName(bucketName) + , s3Helper(profile, region, scheme, endpoint) + { + diskCache = getNarInfoDiskCache(); + } + + std::string getUri() override + { + return "s3://" + bucketName; + } + + void init() override + { + if (!diskCache->cacheExists(getUri(), wantMassQuery_, priority)) { + + BinaryCacheStore::init(); + + diskCache->createCache(getUri(), storeDir, wantMassQuery_, priority); + } + } + + const Stats & getS3Stats() override + { + return stats; + } + + /* This is a specialisation of isValidPath() that optimistically + fetches the .narinfo file, rather than first checking for its + existence via a HEAD request. Since .narinfos are small, doing + a GET is unlikely to be slower than HEAD. */ + bool isValidPathUncached(const Path & storePath) override + { + try { + queryPathInfo(storePath); + return true; + } catch (InvalidPath & e) { + return false; + } + } + + bool fileExists(const std::string & path) override + { + stats.head++; + + auto res = s3Helper.client->HeadObject( + Aws::S3::Model::HeadObjectRequest() + .WithBucket(bucketName) + .WithKey(path)); + + if (!res.IsSuccess()) { + auto & error = res.GetError(); + if (error.GetErrorType() == Aws::S3::S3Errors::RESOURCE_NOT_FOUND + || error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY + // If bucket listing is disabled, 404s turn into 403s + || error.GetErrorType() == Aws::S3::S3Errors::ACCESS_DENIED) + return false; + throw Error(format("AWS error fetching '%s': %s") % path % error.GetMessage()); + } + + return true; + } + + std::shared_ptr transferManager; + std::once_flag transferManagerCreated; + + void uploadFile(const std::string & path, const std::string & data, + const std::string & mimeType, + const std::string & contentEncoding) + { + auto stream = std::make_shared(data); + + auto maxThreads = std::thread::hardware_concurrency(); + + static std::shared_ptr + executor = std::make_shared(maxThreads); + + std::call_once(transferManagerCreated, [&]() + { + if (multipartUpload) { + TransferManagerConfiguration transferConfig(executor.get()); + + transferConfig.s3Client = s3Helper.client; + transferConfig.bufferSize = bufferSize; + + transferConfig.uploadProgressCallback = + [](const TransferManager *transferManager, + const std::shared_ptr + &transferHandle) + { + //FIXME: find a way to properly abort the multipart upload. + //checkInterrupt(); + debug("upload progress ('%s'): '%d' of '%d' bytes", + transferHandle->GetKey(), + transferHandle->GetBytesTransferred(), + transferHandle->GetBytesTotalSize()); + }; + + transferManager = TransferManager::Create(transferConfig); + } + }); + + auto now1 = std::chrono::steady_clock::now(); + + if (transferManager) { + + if (contentEncoding != "") + throw Error("setting a content encoding is not supported with S3 multi-part uploads"); + + std::shared_ptr transferHandle = + transferManager->UploadFile( + stream, bucketName, path, mimeType, + Aws::Map(), + nullptr /*, contentEncoding */); + + transferHandle->WaitUntilFinished(); + + if (transferHandle->GetStatus() == TransferStatus::FAILED) + throw Error("AWS error: failed to upload 's3://%s/%s': %s", + bucketName, path, transferHandle->GetLastError().GetMessage()); + + if (transferHandle->GetStatus() != TransferStatus::COMPLETED) + throw Error("AWS error: transfer status of 's3://%s/%s' in unexpected state", + bucketName, path); + + } else { + + auto request = + Aws::S3::Model::PutObjectRequest() + .WithBucket(bucketName) + .WithKey(path); + + request.SetContentType(mimeType); + + if (contentEncoding != "") + request.SetContentEncoding(contentEncoding); + + auto stream = std::make_shared(data); + + request.SetBody(stream); + + auto result = checkAws(fmt("AWS error uploading '%s'", path), + s3Helper.client->PutObject(request)); + } + + auto now2 = std::chrono::steady_clock::now(); + + auto duration = + std::chrono::duration_cast(now2 - now1) + .count(); + + printInfo(format("uploaded 's3://%1%/%2%' (%3% bytes) in %4% ms") % + bucketName % path % data.size() % duration); + + stats.putTimeMs += duration; + stats.putBytes += data.size(); + stats.put++; + } + + void upsertFile(const std::string & path, const std::string & data, + const std::string & mimeType) override + { + if (narinfoCompression != "" && hasSuffix(path, ".narinfo")) + uploadFile(path, *compress(narinfoCompression, data), mimeType, narinfoCompression); + else if (lsCompression != "" && hasSuffix(path, ".ls")) + uploadFile(path, *compress(lsCompression, data), mimeType, lsCompression); + else if (logCompression != "" && hasPrefix(path, "log/")) + uploadFile(path, *compress(logCompression, data), mimeType, logCompression); + else + uploadFile(path, data, mimeType, ""); + } + + void getFile(const std::string & path, Sink & sink) override + { + stats.get++; + + // FIXME: stream output to sink. + auto res = s3Helper.getObject(bucketName, path); + + stats.getBytes += res.data ? res.data->size() : 0; + stats.getTimeMs += res.durationMs; + + if (res.data) { + printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms", + bucketName, path, res.data->size(), res.durationMs); + + sink((unsigned char *) res.data->data(), res.data->size()); + } else + throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri()); + } + + PathSet queryAllValidPaths() override + { + PathSet paths; + std::string marker; + + do { + debug(format("listing bucket 's3://%s' from key '%s'...") % bucketName % marker); + + auto res = checkAws(format("AWS error listing bucket '%s'") % bucketName, + s3Helper.client->ListObjects( + Aws::S3::Model::ListObjectsRequest() + .WithBucket(bucketName) + .WithDelimiter("/") + .WithMarker(marker))); + + auto & contents = res.GetContents(); + + debug(format("got %d keys, next marker '%s'") + % contents.size() % res.GetNextMarker()); + + for (auto object : contents) { + auto & key = object.GetKey(); + if (key.size() != 40 || !hasSuffix(key, ".narinfo")) continue; + paths.insert(storeDir + "/" + key.substr(0, key.size() - 8)); + } + + marker = res.GetNextMarker(); + } while (!marker.empty()); + + return paths; + } + +}; + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, 5) != "s3://") return 0; + auto store = std::make_shared(params, std::string(uri, 5)); + store->init(); + return store; +}); + +} + +#endif diff --git a/third_party/nix/src/libstore/s3-binary-cache-store.hh b/third_party/nix/src/libstore/s3-binary-cache-store.hh new file mode 100644 index 000000000..4d43fe4d2 --- /dev/null +++ b/third_party/nix/src/libstore/s3-binary-cache-store.hh @@ -0,0 +1,33 @@ +#pragma once + +#include "binary-cache-store.hh" + +#include + +namespace nix { + +class S3BinaryCacheStore : public BinaryCacheStore +{ +protected: + + S3BinaryCacheStore(const Params & params) + : BinaryCacheStore(params) + { } + +public: + + struct Stats + { + std::atomic put{0}; + std::atomic putBytes{0}; + std::atomic putTimeMs{0}; + std::atomic get{0}; + std::atomic getBytes{0}; + std::atomic getTimeMs{0}; + std::atomic head{0}; + }; + + virtual const Stats & getS3Stats() = 0; +}; + +} diff --git a/third_party/nix/src/libstore/s3.hh b/third_party/nix/src/libstore/s3.hh new file mode 100644 index 000000000..ef5f23d0f --- /dev/null +++ b/third_party/nix/src/libstore/s3.hh @@ -0,0 +1,33 @@ +#pragma once + +#if ENABLE_S3 + +#include "ref.hh" + +namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace S3 { class S3Client; } } + +namespace nix { + +struct S3Helper +{ + ref config; + ref client; + + S3Helper(const std::string & profile, const std::string & region, const std::string & scheme, const std::string & endpoint); + + ref makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint); + + struct DownloadResult + { + std::shared_ptr data; + unsigned int durationMs; + }; + + DownloadResult getObject( + const std::string & bucketName, const std::string & key); +}; + +} + +#endif diff --git a/third_party/nix/src/libstore/sandbox-defaults.sb b/third_party/nix/src/libstore/sandbox-defaults.sb new file mode 100644 index 000000000..0299d1ee4 --- /dev/null +++ b/third_party/nix/src/libstore/sandbox-defaults.sb @@ -0,0 +1,87 @@ +(define TMPDIR (param "_GLOBAL_TMP_DIR")) + +(deny default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) + +; Allow forking. +(allow process-fork) + +; Allow reading system information like #CPUs, etc. +(allow sysctl-read) + +; Allow POSIX semaphores and shared memory. +(allow ipc-posix*) + +; Allow socket creation. +(allow system-socket) + +; Allow sending signals within the sandbox. +(allow signal (target same-sandbox)) + +; Allow getpwuid. +(allow mach-lookup (global-name "com.apple.system.opendirectoryd.libinfo")) + +; Access to /tmp. +; The network-outbound/network-inbound ones are for unix domain sockets, which +; we allow access to in TMPDIR (but if we allow them more broadly, you could in +; theory escape the sandbox) +(allow file* process-exec network-outbound network-inbound + (literal "/tmp") (subpath TMPDIR)) + +; Some packages like to read the system version. +(allow file-read* (literal "/System/Library/CoreServices/SystemVersion.plist")) + +; Without this line clang cannot write to /dev/null, breaking some configure tests. +(allow file-read-metadata (literal "/dev")) + +; Many packages like to do local networking in their test suites, but let's only +; allow it if the package explicitly asks for it. +(if (param "_ALLOW_LOCAL_NETWORKING") + (begin + (allow network* (local ip) (local tcp) (local udp)) + + ; Allow access to /etc/resolv.conf (which is a symlink to + ; /private/var/run/resolv.conf). + ; TODO: deduplicate with sandbox-network.sb + (allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + + (allow file-read* + (literal "/private/var/run/resolv.conf")) + + ; Allow DNS lookups. This is even needed for localhost, which lots of tests rely on + (allow file-read-metadata (literal "/etc/hosts")) + (allow file-read* (literal "/private/etc/hosts")) + (allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))))) + +; Standard devices. +(allow file* + (literal "/dev/null") + (literal "/dev/random") + (literal "/dev/stdin") + (literal "/dev/stdout") + (literal "/dev/tty") + (literal "/dev/urandom") + (literal "/dev/zero") + (subpath "/dev/fd")) + +; Does nothing, but reduces build noise. +(allow file* (literal "/dev/dtracehelper")) + +; Allow access to zoneinfo since libSystem needs it. +(allow file-read* (subpath "/usr/share/zoneinfo")) + +(allow file-read* (subpath "/usr/share/locale")) + +; This is mostly to get more specific log messages when builds try to +; access something in /etc or /var. +(allow file-read-metadata + (literal "/etc") + (literal "/var") + (literal "/private/var/tmp")) diff --git a/third_party/nix/src/libstore/sandbox-minimal.sb b/third_party/nix/src/libstore/sandbox-minimal.sb new file mode 100644 index 000000000..65f5108b3 --- /dev/null +++ b/third_party/nix/src/libstore/sandbox-minimal.sb @@ -0,0 +1,5 @@ +(allow default) + +; Disallow creating setuid/setgid binaries, since that +; would allow breaking build user isolation. +(deny file-write-setugid) diff --git a/third_party/nix/src/libstore/sandbox-network.sb b/third_party/nix/src/libstore/sandbox-network.sb new file mode 100644 index 000000000..56beec761 --- /dev/null +++ b/third_party/nix/src/libstore/sandbox-network.sb @@ -0,0 +1,16 @@ +; Allow local and remote network traffic. +(allow network* (local ip) (remote ip)) + +; Allow access to /etc/resolv.conf (which is a symlink to +; /private/var/run/resolv.conf). +(allow file-read-metadata + (literal "/var") + (literal "/etc") + (literal "/etc/resolv.conf") + (literal "/private/etc/resolv.conf")) + +(allow file-read* + (literal "/private/var/run/resolv.conf")) + +; Allow DNS lookups. +(allow network-outbound (remote unix-socket (path-literal "/private/var/run/mDNSResponder"))) diff --git a/third_party/nix/src/libstore/schema.sql b/third_party/nix/src/libstore/schema.sql new file mode 100644 index 000000000..09c71a2b8 --- /dev/null +++ b/third_party/nix/src/libstore/schema.sql @@ -0,0 +1,42 @@ +create table if not exists ValidPaths ( + id integer primary key autoincrement not null, + path text unique not null, + hash text not null, + registrationTime integer not null, + deriver text, + narSize integer, + ultimate integer, -- null implies "false" + sigs text, -- space-separated + ca text -- if not null, an assertion that the path is content-addressed; see ValidPathInfo +); + +create table if not exists Refs ( + referrer integer not null, + reference integer not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict +); + +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger if not exists DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); diff --git a/third_party/nix/src/libstore/serve-protocol.hh b/third_party/nix/src/libstore/serve-protocol.hh new file mode 100644 index 000000000..9fae6d534 --- /dev/null +++ b/third_party/nix/src/libstore/serve-protocol.hh @@ -0,0 +1,24 @@ +#pragma once + +namespace nix { + +#define SERVE_MAGIC_1 0x390c9deb +#define SERVE_MAGIC_2 0x5452eecb + +#define SERVE_PROTOCOL_VERSION 0x205 +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + +typedef enum { + cmdQueryValidPaths = 1, + cmdQueryPathInfos = 2, + cmdDumpStorePath = 3, + cmdImportPaths = 4, + cmdExportPaths = 5, + cmdBuildPaths = 6, + cmdQueryClosure = 7, + cmdBuildDerivation = 8, + cmdAddToStoreNar = 9, +} ServeCommand; + +} diff --git a/third_party/nix/src/libstore/sqlite.cc b/third_party/nix/src/libstore/sqlite.cc new file mode 100644 index 000000000..a061d64f3 --- /dev/null +++ b/third_party/nix/src/libstore/sqlite.cc @@ -0,0 +1,198 @@ +#include "sqlite.hh" +#include "util.hh" + +#include + +#include + +namespace nix { + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs) +{ + int err = sqlite3_errcode(db); + int exterr = sqlite3_extended_errcode(db); + + auto path = sqlite3_db_filename(db, nullptr); + if (!path) path = "(in-memory)"; + + if (err == SQLITE_BUSY || err == SQLITE_PROTOCOL) { + throw SQLiteBusy( + err == SQLITE_PROTOCOL + ? fmt("SQLite database '%s' is busy (SQLITE_PROTOCOL)", path) + : fmt("SQLite database '%s' is busy", path)); + } + else + throw SQLiteError("%s: %s (in '%s')", fs.s, sqlite3_errstr(exterr), path); +} + +SQLite::SQLite(const Path & path) +{ + if (sqlite3_open_v2(path.c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error(format("cannot open SQLite database '%s'") % path); +} + +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throwSQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + +void SQLite::exec(const std::string & stmt) +{ + retrySQLite([&]() { + if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, format("executing SQLite statement '%s'") % stmt); + }); +} + +void SQLiteStmt::create(sqlite3 * db, const string & sql) +{ + checkInterrupt(); + assert(!stmt); + if (sqlite3_prepare_v2(db, sql.c_str(), -1, &stmt, 0) != SQLITE_OK) + throwSQLiteError(db, fmt("creating statement '%s'", sql)); + this->db = db; + this->sql = sql; +} + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throwSQLiteError(db, fmt("finalizing statement '%s'", sql)); + } catch (...) { + ignoreException(); + } +} + +SQLiteStmt::Use::Use(SQLiteStmt & stmt) + : stmt(stmt) +{ + assert(stmt.stmt); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use::~Use() +{ + sqlite3_reset(stmt); +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull) +{ + if (notNull) { + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + } else + bind(); + return *this; +} + +SQLiteStmt::Use & SQLiteStmt::Use::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throwSQLiteError(stmt.db, "binding argument"); + return *this; +} + +int SQLiteStmt::Use::step() +{ + return sqlite3_step(stmt); +} + +void SQLiteStmt::Use::exec() +{ + int r = step(); + assert(r != SQLITE_ROW); + if (r != SQLITE_DONE) + throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql)); +} + +bool SQLiteStmt::Use::next() +{ + int r = step(); + if (r != SQLITE_DONE && r != SQLITE_ROW) + throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql)); + return r == SQLITE_ROW; +} + +std::string SQLiteStmt::Use::getStr(int col) +{ + auto s = (const char *) sqlite3_column_text(stmt, col); + assert(s); + return s; +} + +int64_t SQLiteStmt::Use::getInt(int col) +{ + // FIXME: detect nulls? + return sqlite3_column_int64(stmt, col); +} + +bool SQLiteStmt::Use::isNull(int col) +{ + return sqlite3_column_type(stmt, col) == SQLITE_NULL; +} + +SQLiteTxn::SQLiteTxn(sqlite3 * db) +{ + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "starting transaction"); + active = true; +} + +void SQLiteTxn::commit() +{ + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "committing transaction"); + active = false; +} + +SQLiteTxn::~SQLiteTxn() +{ + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throwSQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } +} + +void handleSQLiteBusy(const SQLiteBusy & e) +{ + static std::atomic lastWarned{0}; + + time_t now = time(0); + + if (now > lastWarned + 10) { + lastWarned = now; + printError("warning: %s", e.what()); + } + + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ + checkInterrupt(); + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ + nanosleep(&t, 0); +} + +} diff --git a/third_party/nix/src/libstore/sqlite.hh b/third_party/nix/src/libstore/sqlite.hh new file mode 100644 index 000000000..115679b84 --- /dev/null +++ b/third_party/nix/src/libstore/sqlite.hh @@ -0,0 +1,114 @@ +#pragma once + +#include +#include + +#include "types.hh" + +class sqlite3; +class sqlite3_stmt; + +namespace nix { + +/* RAII wrapper to close a SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db = 0; + SQLite() { } + SQLite(const Path & path); + SQLite(const SQLite & from) = delete; + SQLite& operator = (const SQLite & from) = delete; + SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; } + ~SQLite(); + operator sqlite3 * () { return db; } + + void exec(const std::string & stmt); +}; + +/* RAII wrapper to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db = 0; + sqlite3_stmt * stmt = 0; + std::string sql; + SQLiteStmt() { } + SQLiteStmt(sqlite3 * db, const std::string & sql) { create(db, sql); } + void create(sqlite3 * db, const std::string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } + + /* Helper for binding / executing statements. */ + class Use + { + friend struct SQLiteStmt; + private: + SQLiteStmt & stmt; + unsigned int curArg = 1; + Use(SQLiteStmt & stmt); + + public: + + ~Use(); + + /* Bind the next parameter. */ + Use & operator () (const std::string & value, bool notNull = true); + Use & operator () (int64_t value, bool notNull = true); + Use & bind(); // null + + int step(); + + /* Execute a statement that does not return rows. */ + void exec(); + + /* For statements that return 0 or more rows. Returns true iff + a row is available. */ + bool next(); + + std::string getStr(int col); + int64_t getInt(int col); + bool isNull(int col); + }; + + Use use() + { + return Use(*this); + } +}; + +/* RAII helper that ensures transactions are aborted unless explicitly + committed. */ +struct SQLiteTxn +{ + bool active = false; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db); + + void commit(); + + ~SQLiteTxn(); +}; + + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + +[[noreturn]] void throwSQLiteError(sqlite3 * db, const FormatOrString & fs); + +void handleSQLiteBusy(const SQLiteBusy & e); + +/* Convenience function for retrying a SQLite transaction when the + database is busy. */ +template +T retrySQLite(std::function fun) +{ + while (true) { + try { + return fun(); + } catch (SQLiteBusy & e) { + handleSQLiteBusy(e); + } + } +} + +} diff --git a/third_party/nix/src/libstore/ssh-store.cc b/third_party/nix/src/libstore/ssh-store.cc new file mode 100644 index 000000000..93e117389 --- /dev/null +++ b/third_party/nix/src/libstore/ssh-store.cc @@ -0,0 +1,100 @@ +#include "store-api.hh" +#include "remote-store.hh" +#include "remote-fs-accessor.hh" +#include "archive.hh" +#include "worker-protocol.hh" +#include "pool.hh" +#include "ssh.hh" + +namespace nix { + +static std::string uriScheme = "ssh-ng://"; + +class SSHStore : public RemoteStore +{ +public: + + const Setting sshKey{(Store*) this, "", "ssh-key", "path to an SSH private key"}; + const Setting compress{(Store*) this, false, "compress", "whether to compress the connection"}; + + SSHStore(const std::string & host, const Params & params) + : Store(params) + , RemoteStore(params) + , host(host) + , master( + host, + sshKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress) + { + } + + std::string getUri() override + { + return uriScheme + host; + } + + bool sameMachine() + { return false; } + + void narFromPath(const Path & path, Sink & sink) override; + + ref getFSAccessor() override; + +private: + + struct Connection : RemoteStore::Connection + { + std::unique_ptr sshConn; + }; + + ref openConnection() override; + + std::string host; + + SSHMaster master; + + void setOptions(RemoteStore::Connection & conn) override + { + /* TODO Add a way to explicitly ask for some options to be + forwarded. One option: A way to query the daemon for its + settings, and then a series of params to SSHStore like + forward-cores or forward-overridden-cores that only + override the requested settings. + */ + }; +}; + +void SSHStore::narFromPath(const Path & path, Sink & sink) +{ + auto conn(connections->get()); + conn->to << wopNarFromPath << path; + conn->processStderr(); + copyNAR(conn->from, sink); +} + +ref SSHStore::getFSAccessor() +{ + return make_ref(ref(shared_from_this())); +} + +ref SSHStore::openConnection() +{ + auto conn = make_ref(); + conn->sshConn = master.startCommand("nix-daemon --stdio"); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + initConnection(*conn); + return conn; +} + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + if (std::string(uri, 0, uriScheme.size()) != uriScheme) return 0; + return std::make_shared(std::string(uri, uriScheme.size()), params); +}); + +} diff --git a/third_party/nix/src/libstore/ssh.cc b/third_party/nix/src/libstore/ssh.cc new file mode 100644 index 000000000..dea16e533 --- /dev/null +++ b/third_party/nix/src/libstore/ssh.cc @@ -0,0 +1,134 @@ +#include "ssh.hh" + +namespace nix { + +SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD) + : host(host) + , fakeSSH(host == "localhost") + , keyFile(keyFile) + , useMaster(useMaster && !fakeSSH) + , compress(compress) + , logFD(logFD) +{ + if (host == "" || hasPrefix(host, "-")) + throw Error("invalid SSH host name '%s'", host); +} + +void SSHMaster::addCommonSSHOpts(Strings & args) +{ + for (auto & i : tokenizeString(getEnv("NIX_SSHOPTS"))) + args.push_back(i); + if (!keyFile.empty()) + args.insert(args.end(), {"-i", keyFile}); + if (compress) + args.push_back("-C"); +} + +std::unique_ptr SSHMaster::startCommand(const std::string & command) +{ + Path socketPath = startMaster(); + + Pipe in, out; + in.create(); + out.create(); + + auto conn = std::make_unique(); + ProcessOptions options; + options.dieWithParent = false; + + conn->sshPid = startProcess([&]() { + restoreSignals(); + + close(in.writeSide.get()); + close(out.readSide.get()); + + if (dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("duping over stdin"); + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("duping over stdout"); + if (logFD != -1 && dup2(logFD, STDERR_FILENO) == -1) + throw SysError("duping over stderr"); + + Strings args; + + if (fakeSSH) { + args = { "bash", "-c" }; + } else { + args = { "ssh", host.c_str(), "-x", "-a" }; + addCommonSSHOpts(args); + if (socketPath != "") + args.insert(args.end(), {"-S", socketPath}); + if (verbosity >= lvlChatty) + args.push_back("-v"); + } + + args.push_back(command); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + + // could not exec ssh/bash + throw SysError("unable to execute '%s'", args.front()); + }, options); + + + in.readSide = -1; + out.writeSide = -1; + + conn->out = std::move(out.readSide); + conn->in = std::move(in.writeSide); + + return conn; +} + +Path SSHMaster::startMaster() +{ + if (!useMaster) return ""; + + auto state(state_.lock()); + + if (state->sshMaster != -1) return state->socketPath; + + state->tmpDir = std::make_unique(createTempDir("", "nix", true, true, 0700)); + + state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; + + Pipe out; + out.create(); + + ProcessOptions options; + options.dieWithParent = false; + + state->sshMaster = startProcess([&]() { + restoreSignals(); + + close(out.readSide.get()); + + if (dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("duping over stdout"); + + Strings args = + { "ssh", host.c_str(), "-M", "-N", "-S", state->socketPath + , "-o", "LocalCommand=echo started" + , "-o", "PermitLocalCommand=yes" + }; + if (verbosity >= lvlChatty) + args.push_back("-v"); + addCommonSSHOpts(args); + execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to execute '%s'", args.front()); + }, options); + + out.writeSide = -1; + + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } + + if (reply != "started") + throw Error("failed to start SSH master connection to '%s'", host); + + return state->socketPath; +} + +} diff --git a/third_party/nix/src/libstore/ssh.hh b/third_party/nix/src/libstore/ssh.hh new file mode 100644 index 000000000..4f0f0bd29 --- /dev/null +++ b/third_party/nix/src/libstore/ssh.hh @@ -0,0 +1,45 @@ +#pragma once + +#include "util.hh" +#include "sync.hh" + +namespace nix { + +class SSHMaster +{ +private: + + const std::string host; + bool fakeSSH; + const std::string keyFile; + const bool useMaster; + const bool compress; + const int logFD; + + struct State + { + Pid sshMaster; + std::unique_ptr tmpDir; + Path socketPath; + }; + + Sync state_; + + void addCommonSSHOpts(Strings & args); + +public: + + SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1); + + struct Connection + { + Pid sshPid; + AutoCloseFD out, in; + }; + + std::unique_ptr startCommand(const std::string & command); + + Path startMaster(); +}; + +} diff --git a/third_party/nix/src/libstore/store-api.cc b/third_party/nix/src/libstore/store-api.cc new file mode 100644 index 000000000..5f63c53b5 --- /dev/null +++ b/third_party/nix/src/libstore/store-api.cc @@ -0,0 +1,983 @@ +#include "crypto.hh" +#include "globals.hh" +#include "store-api.hh" +#include "util.hh" +#include "nar-info-disk-cache.hh" +#include "thread-pool.hh" +#include "json.hh" +#include "derivations.hh" + +#include + + +namespace nix { + + +bool Store::isInStore(const Path & path) const +{ + return isInDir(path, storeDir); +} + + +bool Store::isStorePath(const Path & path) const +{ + return isInStore(path) + && path.size() >= storeDir.size() + 1 + storePathHashLen + && path.find('/', storeDir.size() + 1) == Path::npos; +} + + +void Store::assertStorePath(const Path & path) const +{ + if (!isStorePath(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); +} + + +Path Store::toStorePath(const Path & path) const +{ + if (!isInStore(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); + Path::size_type slash = path.find('/', storeDir.size() + 1); + if (slash == Path::npos) + return path; + else + return Path(path, 0, slash); +} + + +Path Store::followLinksToStore(const Path & _path) const +{ + Path path = absPath(_path); + while (!isInStore(path)) { + if (!isLink(path)) break; + string target = readLink(path); + path = absPath(target, dirOf(path)); + } + if (!isInStore(path)) + throw Error(format("path '%1%' is not in the Nix store") % path); + return path; +} + + +Path Store::followLinksToStorePath(const Path & path) const +{ + return toStorePath(followLinksToStore(path)); +} + + +string storePathToName(const Path & path) +{ + auto base = baseNameOf(path); + assert(base.size() == storePathHashLen || (base.size() > storePathHashLen && base[storePathHashLen] == '-')); + return base.size() == storePathHashLen ? "" : string(base, storePathHashLen + 1); +} + + +string storePathToHash(const Path & path) +{ + auto base = baseNameOf(path); + assert(base.size() >= storePathHashLen); + return string(base, 0, storePathHashLen); +} + + +void checkStoreName(const string & name) +{ + string validChars = "+-._?="; + + auto baseError = format("The path name '%2%' is invalid: %3%. " + "Path names are alphanumeric and can include the symbols %1% " + "and must not begin with a period. " + "Note: If '%2%' is a source file and you cannot rename it on " + "disk, builtins.path { name = ... } can be used to give it an " + "alternative name.") % validChars % name; + + /* Disallow names starting with a dot for possible security + reasons (e.g., "." and ".."). */ + if (string(name, 0, 1) == ".") + throw Error(baseError % "it is illegal to start the name with a period"); + /* Disallow names longer than 211 characters. ext4’s max is 256, + but we need extra space for the hash and .chroot extensions. */ + if (name.length() > 211) + throw Error(baseError % "name must be less than 212 characters"); + for (auto & i : name) + if (!((i >= 'A' && i <= 'Z') || + (i >= 'a' && i <= 'z') || + (i >= '0' && i <= '9') || + validChars.find(i) != string::npos)) + { + throw Error(baseError % (format("the '%1%' character is invalid") % i)); + } +} + + +/* Store paths have the following form: + + /- + + where + + = the location of the Nix store, usually /nix/store + + = a human readable name for the path, typically obtained + from the name attribute of the derivation, or the name of the + source file from which the store path is created. For derivation + outputs other than the default "out" output, the string "-" + is suffixed to . + + = base-32 representation of the first 160 bits of a SHA-256 + hash of ; the hash part of the store name + + = the string ":sha256:

::"; + note that it includes the location of the store as well as the + name to make sure that changes to either of those are reflected + in the hash (e.g. you won't get /nix/store/-name1 and + /nix/store/-name2 with equal hash parts). + + = one of: + "text:::..." + for plain text files written to the store using + addTextToStore(); ... are the references of the + path. + "source" + for paths copied to the store using addToStore() when recursive + = true and hashAlgo = "sha256" + "output:" + for either the outputs created by derivations, OR paths copied + to the store using addToStore() with recursive != true or + hashAlgo != "sha256" (in that case "source" is used; it's + silly, but it's done that way for compatibility). is the + name of the output (usually, "out"). + +

= base-16 representation of a SHA-256 hash of: + if = "text:...": + the string written to the resulting store path + if = "source": + the serialisation of the path from which this store path is + copied, as returned by hashPath() + if = "output:": + for non-fixed derivation outputs: + the derivation (see hashDerivationModulo() in + primops.cc) + for paths copied by addToStore() or produced by fixed-output + derivations: + the string "fixed:out:::", where + = "r:" for recursive (path) hashes, or "" for flat + (file) hashes + = "md5", "sha1" or "sha256" + = base-16 representation of the path or flat hash of + the contents of the path (or expected contents of the + path for fixed-output derivations) + + It would have been nicer to handle fixed-output derivations under + "source", e.g. have something like "source:", but we're + stuck with this for now... + + The main reason for this way of computing names is to prevent name + collisions (for security). For instance, it shouldn't be feasible + to come up with a derivation whose output path collides with the + path for a copied source. The former would have a starting with + "output:out:", while the latter would have a starting with + "source:". +*/ + + +Path Store::makeStorePath(const string & type, + const Hash & hash, const string & name) const +{ + /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ + string s = type + ":" + hash.to_string(Base16) + ":" + storeDir + ":" + name; + + checkStoreName(name); + + return storeDir + "/" + + compressHash(hashString(htSHA256, s), 20).to_string(Base32, false) + + "-" + name; +} + + +Path Store::makeOutputPath(const string & id, + const Hash & hash, const string & name) const +{ + return makeStorePath("output:" + id, hash, + name + (id == "out" ? "" : "-" + id)); +} + + +Path Store::makeFixedOutputPath(bool recursive, + const Hash & hash, const string & name) const +{ + return hash.type == htSHA256 && recursive + ? makeStorePath("source", hash, name) + : makeStorePath("output:out", hashString(htSHA256, + "fixed:out:" + (recursive ? (string) "r:" : "") + + hash.to_string(Base16) + ":"), + name); +} + + +Path Store::makeTextPath(const string & name, const Hash & hash, + const PathSet & references) const +{ + assert(hash.type == htSHA256); + /* Stuff the references (if any) into the type. This is a bit + hacky, but we can't put them in `s' since that would be + ambiguous. */ + string type = "text"; + for (auto & i : references) { + type += ":"; + type += i; + } + return makeStorePath(type, hash, name); +} + + +std::pair Store::computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) const +{ + Hash h = recursive ? hashPath(hashAlgo, srcPath, filter).first : hashFile(hashAlgo, srcPath); + Path dstPath = makeFixedOutputPath(recursive, h, name); + return std::pair(dstPath, h); +} + + +Path Store::computeStorePathForText(const string & name, const string & s, + const PathSet & references) const +{ + return makeTextPath(name, hashString(htSHA256, s), references); +} + + +Store::Store(const Params & params) + : Config(params) + , state({(size_t) pathInfoCacheSize}) +{ +} + + +std::string Store::getUri() +{ + return ""; +} + + +bool Store::isValidPath(const Path & storePath) +{ + assertStorePath(storePath); + + auto hashPart = storePathToHash(storePath); + + { + auto state_(state.lock()); + auto res = state_->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + return *res != 0; + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + return res.first == NarInfoDiskCache::oValid; + } + } + + bool valid = isValidPathUncached(storePath); + + if (diskCache && !valid) + // FIXME: handle valid = true case. + diskCache->upsertNarInfo(getUri(), hashPart, 0); + + return valid; +} + + +/* Default implementation for stores that only implement + queryPathInfoUncached(). */ +bool Store::isValidPathUncached(const Path & path) +{ + try { + queryPathInfo(path); + return true; + } catch (InvalidPath &) { + return false; + } +} + + +ref Store::queryPathInfo(const Path & storePath) +{ + std::promise> promise; + + queryPathInfo(storePath, + {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} + + +void Store::queryPathInfo(const Path & storePath, + Callback> callback) noexcept +{ + std::string hashPart; + + try { + assertStorePath(storePath); + + hashPart = storePathToHash(storePath); + + { + auto res = state.lock()->pathInfoCache.get(hashPart); + if (res) { + stats.narInfoReadAverted++; + if (!*res) + throw InvalidPath(format("path '%s' is not valid") % storePath); + return callback(ref(*res)); + } + } + + if (diskCache) { + auto res = diskCache->lookupNarInfo(getUri(), hashPart); + if (res.first != NarInfoDiskCache::oUnknown) { + stats.narInfoReadAverted++; + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, + res.first == NarInfoDiskCache::oInvalid ? 0 : res.second); + if (res.first == NarInfoDiskCache::oInvalid || + (res.second->path != storePath && storePathToName(storePath) != "")) + throw InvalidPath(format("path '%s' is not valid") % storePath); + } + return callback(ref(res.second)); + } + } + + } catch (...) { return callback.rethrow(); } + + auto callbackPtr = std::make_shared(std::move(callback)); + + queryPathInfoUncached(storePath, + {[this, storePath, hashPart, callbackPtr](std::future> fut) { + + try { + auto info = fut.get(); + + if (diskCache) + diskCache->upsertNarInfo(getUri(), hashPart, info); + + { + auto state_(state.lock()); + state_->pathInfoCache.upsert(hashPart, info); + } + + if (!info + || (info->path != storePath && storePathToName(storePath) != "")) + { + stats.narInfoMissing++; + throw InvalidPath("path '%s' is not valid", storePath); + } + + (*callbackPtr)(ref(info)); + } catch (...) { callbackPtr->rethrow(); } + }}); +} + + +PathSet Store::queryValidPaths(const PathSet & paths, SubstituteFlag maybeSubstitute) +{ + struct State + { + size_t left; + PathSet valid; + std::exception_ptr exc; + }; + + Sync state_(State{paths.size(), PathSet()}); + + std::condition_variable wakeup; + ThreadPool pool; + + auto doQuery = [&](const Path & path ) { + checkInterrupt(); + queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { + auto state(state_.lock()); + try { + auto info = fut.get(); + state->valid.insert(path); + } catch (InvalidPath &) { + } catch (...) { + state->exc = std::current_exception(); + } + assert(state->left); + if (!--state->left) + wakeup.notify_one(); + }}); + }; + + for (auto & path : paths) + pool.enqueue(std::bind(doQuery, path)); + + pool.process(); + + while (true) { + auto state(state_.lock()); + if (!state->left) { + if (state->exc) std::rethrow_exception(state->exc); + return state->valid; + } + state.wait(wakeup); + } +} + + +/* Return a string accepted by decodeValidPathInfo() that + registers the specified paths as valid. Note: it's the + responsibility of the caller to provide a closure. */ +string Store::makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash) +{ + string s = ""; + + for (auto & i : paths) { + s += i + "\n"; + + auto info = queryPathInfo(i); + + if (showHash) { + s += info->narHash.to_string(Base16, false) + "\n"; + s += (format("%1%\n") % info->narSize).str(); + } + + Path deriver = showDerivers ? info->deriver : ""; + s += deriver + "\n"; + + s += (format("%1%\n") % info->references.size()).str(); + + for (auto & j : info->references) + s += j + "\n"; + } + + return s; +} + + +void Store::pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, + bool includeImpureInfo, bool showClosureSize, AllowInvalidFlag allowInvalid) +{ + auto jsonList = jsonOut.list(); + + for (auto storePath : storePaths) { + auto jsonPath = jsonList.object(); + jsonPath.attr("path", storePath); + + try { + auto info = queryPathInfo(storePath); + storePath = info->path; + + jsonPath + .attr("narHash", info->narHash.to_string()) + .attr("narSize", info->narSize); + + { + auto jsonRefs = jsonPath.list("references"); + for (auto & ref : info->references) + jsonRefs.elem(ref); + } + + if (info->ca != "") + jsonPath.attr("ca", info->ca); + + std::pair closureSizes; + + if (showClosureSize) { + closureSizes = getClosureSize(storePath); + jsonPath.attr("closureSize", closureSizes.first); + } + + if (includeImpureInfo) { + + if (info->deriver != "") + jsonPath.attr("deriver", info->deriver); + + if (info->registrationTime) + jsonPath.attr("registrationTime", info->registrationTime); + + if (info->ultimate) + jsonPath.attr("ultimate", info->ultimate); + + if (!info->sigs.empty()) { + auto jsonSigs = jsonPath.list("signatures"); + for (auto & sig : info->sigs) + jsonSigs.elem(sig); + } + + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + + if (narInfo) { + if (!narInfo->url.empty()) + jsonPath.attr("url", narInfo->url); + if (narInfo->fileHash) + jsonPath.attr("downloadHash", narInfo->fileHash.to_string()); + if (narInfo->fileSize) + jsonPath.attr("downloadSize", narInfo->fileSize); + if (showClosureSize) + jsonPath.attr("closureDownloadSize", closureSizes.second); + } + } + + } catch (InvalidPath &) { + jsonPath.attr("valid", false); + } + } +} + + +std::pair Store::getClosureSize(const Path & storePath) +{ + uint64_t totalNarSize = 0, totalDownloadSize = 0; + PathSet closure; + computeFSClosure(storePath, closure, false, false); + for (auto & p : closure) { + auto info = queryPathInfo(p); + totalNarSize += info->narSize; + auto narInfo = std::dynamic_pointer_cast( + std::shared_ptr(info)); + if (narInfo) + totalDownloadSize += narInfo->fileSize; + } + return {totalNarSize, totalDownloadSize}; +} + + +const Store::Stats & Store::getStats() +{ + { + auto state_(state.lock()); + stats.pathInfoCacheSize = state_->pathInfoCache.size(); + } + return stats; +} + + +void Store::buildPaths(const PathSet & paths, BuildMode buildMode) +{ + for (auto & path : paths) + if (isDerivation(path)) + unsupported("buildPaths"); + + if (queryValidPaths(paths).size() != paths.size()) + unsupported("buildPaths"); +} + + +void copyStorePath(ref srcStore, ref dstStore, + const Path & storePath, RepairFlag repair, CheckSigsFlag checkSigs) +{ + auto srcUri = srcStore->getUri(); + auto dstUri = dstStore->getUri(); + + Activity act(*logger, lvlInfo, actCopyPath, + srcUri == "local" || srcUri == "daemon" + ? fmt("copying path '%s' to '%s'", storePath, dstUri) + : dstUri == "local" || dstUri == "daemon" + ? fmt("copying path '%s' from '%s'", storePath, srcUri) + : fmt("copying path '%s' from '%s' to '%s'", storePath, srcUri, dstUri), + {storePath, srcUri, dstUri}); + PushActivity pact(act.id); + + auto info = srcStore->queryPathInfo(storePath); + + uint64_t total = 0; + + if (!info->narHash) { + StringSink sink; + srcStore->narFromPath({storePath}, sink); + auto info2 = make_ref(*info); + info2->narHash = hashString(htSHA256, *sink.s); + if (!info->narSize) info2->narSize = sink.s->size(); + if (info->ultimate) info2->ultimate = false; + info = info2; + + StringSource source(*sink.s); + dstStore->addToStore(*info, source, repair, checkSigs); + return; + } + + if (info->ultimate) { + auto info2 = make_ref(*info); + info2->ultimate = false; + info = info2; + } + + auto source = sinkToSource([&](Sink & sink) { + LambdaSink wrapperSink([&](const unsigned char * data, size_t len) { + sink(data, len); + total += len; + act.progress(total, info->narSize); + }); + srcStore->narFromPath({storePath}, wrapperSink); + }, [&]() { + throw EndOfFile("NAR for '%s' fetched from '%s' is incomplete", storePath, srcStore->getUri()); + }); + + dstStore->addToStore(*info, *source, repair, checkSigs); +} + + +void copyPaths(ref srcStore, ref dstStore, const PathSet & storePaths, + RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) +{ + PathSet valid = dstStore->queryValidPaths(storePaths, substitute); + + PathSet missing; + for (auto & path : storePaths) + if (!valid.count(path)) missing.insert(path); + + if (missing.empty()) return; + + Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size())); + + std::atomic nrDone{0}; + std::atomic nrFailed{0}; + std::atomic bytesExpected{0}; + std::atomic nrRunning{0}; + + auto showProgress = [&]() { + act.progress(nrDone, missing.size(), nrRunning, nrFailed); + }; + + ThreadPool pool; + + processGraph(pool, + PathSet(missing.begin(), missing.end()), + + [&](const Path & storePath) { + if (dstStore->isValidPath(storePath)) { + nrDone++; + showProgress(); + return PathSet(); + } + + auto info = srcStore->queryPathInfo(storePath); + + bytesExpected += info->narSize; + act.setExpected(actCopyPath, bytesExpected); + + return info->references; + }, + + [&](const Path & storePath) { + checkInterrupt(); + + if (!dstStore->isValidPath(storePath)) { + MaintainCount mc(nrRunning); + showProgress(); + try { + copyStorePath(srcStore, dstStore, storePath, repair, checkSigs); + } catch (Error &e) { + nrFailed++; + if (!settings.keepGoing) + throw e; + logger->log(lvlError, format("could not copy %s: %s") % storePath % e.what()); + showProgress(); + return; + } + } + + nrDone++; + showProgress(); + }); +} + + +void copyClosure(ref srcStore, ref dstStore, + const PathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs, + SubstituteFlag substitute) +{ + PathSet closure; + srcStore->computeFSClosure({storePaths}, closure); + copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute); +} + + +ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) +{ + ValidPathInfo info; + getline(str, info.path); + if (str.eof()) { info.path = ""; return info; } + if (hashGiven) { + string s; + getline(str, s); + info.narHash = Hash(s, htSHA256); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); + } + getline(str, info.deriver); + string s; int n; + getline(str, s); + if (!string2Int(s, n)) throw Error("number expected"); + while (n--) { + getline(str, s); + info.references.insert(s); + } + if (!str || str.eof()) throw Error("missing input"); + return info; +} + + +string showPaths(const PathSet & paths) +{ + string s; + for (auto & i : paths) { + if (s.size() != 0) s += ", "; + s += "'" + i + "'"; + } + return s; +} + + +std::string ValidPathInfo::fingerprint() const +{ + if (narSize == 0 || !narHash) + throw Error(format("cannot calculate fingerprint of path '%s' because its size/hash is not known") + % path); + return + "1;" + path + ";" + + narHash.to_string(Base32) + ";" + + std::to_string(narSize) + ";" + + concatStringsSep(",", references); +} + + +void ValidPathInfo::sign(const SecretKey & secretKey) +{ + sigs.insert(secretKey.signDetached(fingerprint())); +} + + +bool ValidPathInfo::isContentAddressed(const Store & store) const +{ + auto warn = [&]() { + printError(format("warning: path '%s' claims to be content-addressed but isn't") % path); + }; + + if (hasPrefix(ca, "text:")) { + Hash hash(std::string(ca, 5)); + if (store.makeTextPath(storePathToName(path), hash, references) == path) + return true; + else + warn(); + } + + else if (hasPrefix(ca, "fixed:")) { + bool recursive = ca.compare(6, 2, "r:") == 0; + Hash hash(std::string(ca, recursive ? 8 : 6)); + if (references.empty() && + store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path) + return true; + else + warn(); + } + + return false; +} + + +size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const +{ + if (isContentAddressed(store)) return maxSigs; + + size_t good = 0; + for (auto & sig : sigs) + if (checkSignature(publicKeys, sig)) + good++; + return good; +} + + +bool ValidPathInfo::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const +{ + return verifyDetached(fingerprint(), sig, publicKeys); +} + + +Strings ValidPathInfo::shortRefs() const +{ + Strings refs; + for (auto & r : references) + refs.push_back(baseNameOf(r)); + return refs; +} + + +std::string makeFixedOutputCA(bool recursive, const Hash & hash) +{ + return "fixed:" + (recursive ? (std::string) "r:" : "") + hash.to_string(); +} + + +void Store::addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) +{ + addToStore(info, make_ref(narSource.drain()), repair, checkSigs, accessor); +} + +void Store::addToStore(const ValidPathInfo & info, const ref & nar, + RepairFlag repair, CheckSigsFlag checkSigs, + std::shared_ptr accessor) +{ + StringSource source(*nar); + addToStore(info, source, repair, checkSigs, accessor); +} + +} + + +#include "local-store.hh" +#include "remote-store.hh" + + +namespace nix { + + +RegisterStoreImplementation::Implementations * RegisterStoreImplementation::implementations = 0; + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) +{ + auto uri(uri_); + Store::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + for (auto s : tokenizeString(uri.substr(q + 1), "&")) { + auto e = s.find('='); + if (e != std::string::npos) { + auto value = s.substr(e + 1); + std::string decoded; + for (size_t i = 0; i < value.size(); ) { + if (value[i] == '%') { + if (i + 2 >= value.size()) + throw Error("invalid URI parameter '%s'", value); + try { + decoded += std::stoul(std::string(value, i + 1, 2), 0, 16); + i += 3; + } catch (...) { + throw Error("invalid URI parameter '%s'", value); + } + } else + decoded += value[i++]; + } + params[s.substr(0, e)] = decoded; + } + } + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +ref openStore(const std::string & uri_, + const Store::Params & extraParams) +{ + auto [uri, uriParams] = splitUriAndParams(uri_); + auto params = extraParams; + params.insert(uriParams.begin(), uriParams.end()); + + for (auto fun : *RegisterStoreImplementation::implementations) { + auto store = fun(uri, params); + if (store) { + store->warnUnknownSettings(); + return ref(store); + } + } + + throw Error("don't know how to open Nix store '%s'", uri); +} + + +StoreType getStoreType(const std::string & uri, const std::string & stateDir) +{ + if (uri == "daemon") { + return tDaemon; + } else if (uri == "local" || hasPrefix(uri, "/")) { + return tLocal; + } else if (uri == "" || uri == "auto") { + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return tLocal; + else if (pathExists(settings.nixDaemonSocketFile)) + return tDaemon; + else + return tLocal; + } else { + return tOther; + } +} + + +static RegisterStoreImplementation regStore([]( + const std::string & uri, const Store::Params & params) + -> std::shared_ptr +{ + switch (getStoreType(uri, get(params, "state", settings.nixStateDir))) { + case tDaemon: + return std::shared_ptr(std::make_shared(params)); + case tLocal: { + Store::Params params2 = params; + if (hasPrefix(uri, "/")) + params2["root"] = uri; + return std::shared_ptr(std::make_shared(params2)); + } + default: + return nullptr; + } +}); + + +std::list> getDefaultSubstituters() +{ + static auto stores([]() { + std::list> stores; + + StringSet done; + + auto addStore = [&](const std::string & uri) { + if (done.count(uri)) return; + done.insert(uri); + try { + stores.push_back(openStore(uri)); + } catch (Error & e) { + printError("warning: %s", e.what()); + } + }; + + for (auto uri : settings.substituters.get()) + addStore(uri); + + for (auto uri : settings.extraSubstituters.get()) + addStore(uri); + + stores.sort([](ref & a, ref & b) { + return a->getPriority() < b->getPriority(); + }); + + return stores; + } ()); + + return stores; +} + + +} diff --git a/third_party/nix/src/libstore/store-api.hh b/third_party/nix/src/libstore/store-api.hh new file mode 100644 index 000000000..a751b4662 --- /dev/null +++ b/third_party/nix/src/libstore/store-api.hh @@ -0,0 +1,811 @@ +#pragma once + +#include "hash.hh" +#include "serialise.hh" +#include "crypto.hh" +#include "lru-cache.hh" +#include "sync.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include +#include +#include +#include +#include + + +namespace nix { + + +MakeError(SubstError, Error) +MakeError(BuildError, Error) /* denotes a permanent build failure */ +MakeError(InvalidPath, Error) +MakeError(Unsupported, Error) +MakeError(SubstituteGone, Error) +MakeError(SubstituterDisabled, Error) + + +struct BasicDerivation; +struct Derivation; +class FSAccessor; +class NarInfoDiskCache; +class Store; +class JSONPlaceholder; + + +enum RepairFlag : bool { NoRepair = false, Repair = true }; +enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; +enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; +enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; + + +/* Size of the hash part of store paths, in base-32 characters. */ +const size_t storePathHashLen = 32; // i.e. 160 bits + +/* Magic header of exportPath() output (obsolete). */ +const uint32_t exportMagic = 0x4558494e; + + +typedef std::unordered_map> Roots; + + +struct GCOptions +{ + /* Garbage collector operation: + + - `gcReturnLive': return the set of paths reachable from + (i.e. in the closure of) the roots. + + - `gcReturnDead': return the set of paths not reachable from + the roots. + + - `gcDeleteDead': actually delete the latter set. + + - `gcDeleteSpecific': delete the paths listed in + `pathsToDelete', insofar as they are not reachable. + */ + typedef enum { + gcReturnLive, + gcReturnDead, + gcDeleteDead, + gcDeleteSpecific, + } GCAction; + + GCAction action{gcDeleteDead}; + + /* If `ignoreLiveness' is set, then reachability from the roots is + ignored (dangerous!). However, the paths must still be + unreferenced *within* the store (i.e., there can be no other + store paths that depend on them). */ + bool ignoreLiveness{false}; + + /* For `gcDeleteSpecific', the paths to delete. */ + PathSet pathsToDelete; + + /* Stop after at least `maxFreed' bytes have been freed. */ + unsigned long long maxFreed{std::numeric_limits::max()}; +}; + + +struct GCResults +{ + /* Depending on the action, the GC roots, or the paths that would + be or have been deleted. */ + PathSet paths; + + /* For `gcReturnDead', `gcDeleteDead' and `gcDeleteSpecific', the + number of bytes that would be or was freed. */ + unsigned long long bytesFreed = 0; +}; + + +struct SubstitutablePathInfo +{ + Path deriver; + PathSet references; + unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ +}; + +typedef std::map SubstitutablePathInfos; + + +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash narHash; + PathSet references; + time_t registrationTime = 0; + uint64_t narSize = 0; // 0 = unknown + uint64_t id; // internal use only + + /* Whether the path is ultimately trusted, that is, it's a + derivation output that was built locally. */ + bool ultimate = false; + + StringSet sigs; // note: not necessarily verified + + /* If non-empty, an assertion that the path is content-addressed, + i.e., that the store path is computed from a cryptographic hash + of the contents of the path, plus some other bits of data like + the "name" part of the path. Such a path doesn't need + signatures, since we don't have to trust anybody's claim that + the path is the output of a particular derivation. (In the + extensional store model, we have to trust that the *contents* + of an output path of a derivation were actually produced by + that derivation. In the intensional model, we have to trust + that a particular output path was produced by a derivation; the + path then implies the contents.) + + Ideally, the content-addressability assertion would just be a + Boolean, and the store path would be computed from + ‘storePathToName(path)’, ‘narHash’ and ‘references’. However, + 1) we've accumulated several types of content-addressed paths + over the years; and 2) fixed-output derivations support + multiple hash algorithms and serialisation methods (flat file + vs NAR). Thus, ‘ca’ has one of the following forms: + + * ‘text:sha256:’: For paths + computed by makeTextPath() / addTextToStore(). + + * ‘fixed:::’: For paths computed by + makeFixedOutputPath() / addToStore(). + */ + std::string ca; + + bool operator == (const ValidPathInfo & i) const + { + return + path == i.path + && narHash == i.narHash + && references == i.references; + } + + /* Return a fingerprint of the store path to be used in binary + cache signatures. It contains the store path, the base-32 + SHA-256 hash of the NAR serialisation of the path, the size of + the NAR, and the sorted references. The size field is strictly + speaking superfluous, but might prevent endless/excessive data + attacks. */ + std::string fingerprint() const; + + void sign(const SecretKey & secretKey); + + /* Return true iff the path is verifiably content-addressed. */ + bool isContentAddressed(const Store & store) const; + + static const size_t maxSigs = std::numeric_limits::max(); + + /* Return the number of signatures on this .narinfo that were + produced by one of the specified keys, or maxSigs if the path + is content-addressed. */ + size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const; + + /* Verify a single signature. */ + bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; + + Strings shortRefs() const; + + virtual ~ValidPathInfo() { } +}; + +typedef list ValidPathInfos; + + +enum BuildMode { bmNormal, bmRepair, bmCheck }; + + +struct BuildResult +{ + /* Note: don't remove status codes, and only add new status codes + at the end of the list, to prevent client/server + incompatibilities in the nix-store --serve protocol. */ + enum Status { + Built = 0, + Substituted, + AlreadyValid, + PermanentFailure, + InputRejected, + OutputRejected, + TransientFailure, // possibly transient + CachedFailure, // no longer used + TimedOut, + MiscFailure, + DependencyFailed, + LogLimitExceeded, + NotDeterministic, + } status = MiscFailure; + std::string errorMsg; + + /* How many times this build was performed. */ + unsigned int timesBuilt = 0; + + /* If timesBuilt > 1, whether some builds did not produce the same + result. (Note that 'isNonDeterministic = false' does not mean + the build is deterministic, just that we don't have evidence of + non-determinism.) */ + bool isNonDeterministic = false; + + /* The start/stop times of the build (or one of the rounds, if it + was repeated). */ + time_t startTime = 0, stopTime = 0; + + bool success() { + return status == Built || status == Substituted || status == AlreadyValid; + } +}; + + +class Store : public std::enable_shared_from_this, public Config +{ +public: + + typedef std::map Params; + + const PathSetting storeDir_{this, false, settings.nixStore, + "store", "path to the Nix store"}; + const Path storeDir = storeDir_; + + const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "size of the in-memory store path information cache"}; + + const Setting isTrusted{this, false, "trusted", "whether paths from this store can be used as substitutes even when they lack trusted signatures"}; + +protected: + + struct State + { + LRUCache> pathInfoCache; + }; + + Sync state; + + std::shared_ptr diskCache; + + Store(const Params & params); + +public: + + virtual ~Store() { } + + virtual std::string getUri() = 0; + + /* Return true if ‘path’ is in the Nix store (but not the Nix + store itself). */ + bool isInStore(const Path & path) const; + + /* Return true if ‘path’ is a store path, i.e. a direct child of + the Nix store. */ + bool isStorePath(const Path & path) const; + + /* Throw an exception if ‘path’ is not a store path. */ + void assertStorePath(const Path & path) const; + + /* Chop off the parts after the top-level store name, e.g., + /nix/store/abcd-foo/bar => /nix/store/abcd-foo. */ + Path toStorePath(const Path & path) const; + + /* Follow symlinks until we end up with a path in the Nix store. */ + Path followLinksToStore(const Path & path) const; + + /* Same as followLinksToStore(), but apply toStorePath() to the + result. */ + Path followLinksToStorePath(const Path & path) const; + + /* Constructs a unique store path name. */ + Path makeStorePath(const string & type, + const Hash & hash, const string & name) const; + + Path makeOutputPath(const string & id, + const Hash & hash, const string & name) const; + + Path makeFixedOutputPath(bool recursive, + const Hash & hash, const string & name) const; + + Path makeTextPath(const string & name, const Hash & hash, + const PathSet & references) const; + + /* This is the preparatory part of addToStore(); it computes the + store path to which srcPath is to be copied. Returns the store + path and the cryptographic hash of the contents of srcPath. */ + std::pair computeStorePathForPath(const string & name, + const Path & srcPath, bool recursive = true, + HashType hashAlgo = htSHA256, PathFilter & filter = defaultPathFilter) const; + + /* Preparatory part of addTextToStore(). + + !!! Computation of the path should take the references given to + addTextToStore() into account, otherwise we have a (relatively + minor) security hole: a caller can register a source file with + bogus references. If there are too many references, the path may + not be garbage collected when it has to be (not really a problem, + the caller could create a root anyway), or it may be garbage + collected when it shouldn't be (more serious). + + Hashing the references would solve this (bogus references would + simply yield a different store path, so other users wouldn't be + affected), but it has some backwards compatibility issues (the + hashing scheme changes), so I'm not doing that for now. */ + Path computeStorePathForText(const string & name, const string & s, + const PathSet & references) const; + + /* Check whether a path is valid. */ + bool isValidPath(const Path & path); + +protected: + + virtual bool isValidPathUncached(const Path & path); + +public: + + /* Query which of the given paths is valid. Optionally, try to + substitute missing paths. */ + virtual PathSet queryValidPaths(const PathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute); + + /* Query the set of all valid paths. Note that for some store + backends, the name part of store paths may be omitted + (i.e. you'll get /nix/store/ rather than + /nix/store/-). Use queryPathInfo() to obtain the + full store path. */ + virtual PathSet queryAllValidPaths() + { unsupported("queryAllValidPaths"); } + + /* Query information about a valid path. It is permitted to omit + the name part of the store path. */ + ref queryPathInfo(const Path & path); + + /* Asynchronous version of queryPathInfo(). */ + void queryPathInfo(const Path & path, + Callback> callback) noexcept; + +protected: + + virtual void queryPathInfoUncached(const Path & path, + Callback> callback) noexcept = 0; + +public: + + /* Queries the set of incoming FS references for a store path. + The result is not cleared. */ + virtual void queryReferrers(const Path & path, PathSet & referrers) + { unsupported("queryReferrers"); } + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + virtual PathSet queryValidDerivers(const Path & path) { return {}; }; + + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path & path) + { unsupported("queryDerivationOutputs"); } + + /* Query the output names of the derivation denoted by `path'. */ + virtual StringSet queryDerivationOutputNames(const Path & path) + { unsupported("queryDerivationOutputNames"); } + + /* Query the full store path given the hash part of a valid store + path, or "" if the path doesn't exist. */ + virtual Path queryPathFromHashPart(const string & hashPart) = 0; + + /* Query which of the given paths have substitutes. */ + virtual PathSet querySubstitutablePaths(const PathSet & paths) { return {}; }; + + /* Query substitute info (i.e. references, derivers and download + sizes) of a set of paths. If a path does not have substitute + info, it's omitted from the resulting ‘infos’ map. */ + virtual void querySubstitutablePathInfos(const PathSet & paths, + SubstitutablePathInfos & infos) { return; }; + + virtual bool wantMassQuery() { return false; } + + /* Import a path into the store. */ + virtual void addToStore(const ValidPathInfo & info, Source & narSource, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0); + + // FIXME: remove + virtual void addToStore(const ValidPathInfo & info, const ref & nar, + RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, + std::shared_ptr accessor = 0); + + /* Copy the contents of a path to the store and register the + validity the resulting path. The resulting path is returned. + The function object `filter' can be used to exclude files (see + libutil/archive.hh). */ + virtual Path addToStore(const string & name, const Path & srcPath, + bool recursive = true, HashType hashAlgo = htSHA256, + PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) = 0; + + /* Like addToStore, but the contents written to the output path is + a regular file containing the given string. */ + virtual Path addTextToStore(const string & name, const string & s, + const PathSet & references, RepairFlag repair = NoRepair) = 0; + + /* Write a NAR dump of a store path. */ + virtual void narFromPath(const Path & path, Sink & sink) = 0; + + /* For each path, if it's a derivation, build it. Building a + derivation means ensuring that the output paths are valid. If + they are already valid, this is a no-op. Otherwise, validity + can be reached in two ways. First, if the output paths is + substitutable, then build the path that way. Second, the + output paths can be created by running the builder, after + recursively building any sub-derivations. For inputs that are + not derivations, substitute them. */ + virtual void buildPaths(const PathSet & paths, BuildMode buildMode = bmNormal); + + /* Build a single non-materialized derivation (i.e. not from an + on-disk .drv file). Note that ‘drvPath’ is only used for + informational purposes. */ + virtual BuildResult buildDerivation(const Path & drvPath, const BasicDerivation & drv, + BuildMode buildMode = bmNormal) = 0; + + /* Ensure that a path is valid. If it is not currently valid, it + may be made valid by running a substitute (if defined for the + path). */ + virtual void ensurePath(const Path & path) = 0; + + /* Add a store path as a temporary root of the garbage collector. + The root disappears as soon as we exit. */ + virtual void addTempRoot(const Path & path) + { unsupported("addTempRoot"); } + + /* Add an indirect root, which is merely a symlink to `path' from + /nix/var/nix/gcroots/auto/. `path' is supposed + to be a symlink to a store path. The garbage collector will + automatically remove the indirect root when it finds that + `path' has disappeared. */ + virtual void addIndirectRoot(const Path & path) + { unsupported("addIndirectRoot"); } + + /* Acquire the global GC lock, then immediately release it. This + function must be called after registering a new permanent root, + but before exiting. Otherwise, it is possible that a running + garbage collector doesn't see the new root and deletes the + stuff we've just built. By acquiring the lock briefly, we + ensure that either: + + - The collector is already running, and so we block until the + collector is finished. The collector will know about our + *temporary* locks, which should include whatever it is we + want to register as a permanent lock. + + - The collector isn't running, or it's just started but hasn't + acquired the GC lock yet. In that case we get and release + the lock right away, then exit. The collector scans the + permanent root and sees our's. + + In either case the permanent root is seen by the collector. */ + virtual void syncWithGC() { }; + + /* Find the roots of the garbage collector. Each root is a pair + (link, storepath) where `link' is the path of the symlink + outside of the Nix store that point to `storePath'. If + 'censor' is true, privacy-sensitive information about roots + found in /proc is censored. */ + virtual Roots findRoots(bool censor) + { unsupported("findRoots"); } + + /* Perform a garbage collection. */ + virtual void collectGarbage(const GCOptions & options, GCResults & results) + { unsupported("collectGarbage"); } + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); + + /* Write a JSON representation of store path metadata, such as the + hash and the references. If ‘includeImpureInfo’ is true, + variable elements such as the registration time are + included. If ‘showClosureSize’ is true, the closure size of + each path is included. */ + void pathInfoToJSON(JSONPlaceholder & jsonOut, const PathSet & storePaths, + bool includeImpureInfo, bool showClosureSize, + AllowInvalidFlag allowInvalid = DisallowInvalid); + + /* Return the size of the closure of the specified path, that is, + the sum of the size of the NAR serialisation of each path in + the closure. */ + std::pair getClosureSize(const Path & storePath); + + /* Optimise the disk space usage of the Nix store by hard-linking files + with the same contents. */ + virtual void optimiseStore() { }; + + /* Check the integrity of the Nix store. Returns true if errors + remain. */ + virtual bool verifyStore(bool checkContents, RepairFlag repair = NoRepair) { return false; }; + + /* Return an object to access files in the Nix store. */ + virtual ref getFSAccessor() + { unsupported("getFSAccessor"); } + + /* Add signatures to the specified store path. The signatures are + not verified. */ + virtual void addSignatures(const Path & storePath, const StringSet & sigs) + { unsupported("addSignatures"); } + + /* Utility functions. */ + + /* Read a derivation, after ensuring its existence through + ensurePath(). */ + Derivation derivationFromPath(const Path & drvPath); + + /* Place in `out' the set of all store paths in the file system + closure of `storePath'; that is, all paths than can be directly + or indirectly reached from it. `out' is not cleared. If + `flipDirection' is true, the set of paths that can reach + `storePath' is returned; that is, the closures under the + `referrers' relation instead of the `references' relation is + returned. */ + virtual void computeFSClosure(const PathSet & paths, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + + void computeFSClosure(const Path & path, + PathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false); + + /* Given a set of paths that are to be built, return the set of + derivations that will be built, and the set of output paths + that will be substituted. */ + virtual void queryMissing(const PathSet & targets, + PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, + unsigned long long & downloadSize, unsigned long long & narSize); + + /* Sort a set of paths topologically under the references + relation. If p refers to q, then p precedes q in this list. */ + Paths topoSortPaths(const PathSet & paths); + + /* Export multiple paths in the format expected by ‘nix-store + --import’. */ + void exportPaths(const Paths & paths, Sink & sink); + + void exportPath(const Path & path, Sink & sink); + + /* Import a sequence of NAR dumps created by exportPaths() into + the Nix store. Optionally, the contents of the NARs are + preloaded into the specified FS accessor to speed up subsequent + access. */ + Paths importPaths(Source & source, std::shared_ptr accessor, + CheckSigsFlag checkSigs = CheckSigs); + + struct Stats + { + std::atomic narInfoRead{0}; + std::atomic narInfoReadAverted{0}; + std::atomic narInfoMissing{0}; + std::atomic narInfoWrite{0}; + std::atomic pathInfoCacheSize{0}; + std::atomic narRead{0}; + std::atomic narReadBytes{0}; + std::atomic narReadCompressedBytes{0}; + std::atomic narWrite{0}; + std::atomic narWriteAverted{0}; + std::atomic narWriteBytes{0}; + std::atomic narWriteCompressedBytes{0}; + std::atomic narWriteCompressionTimeMs{0}; + }; + + const Stats & getStats(); + + /* Return the build log of the specified store path, if available, + or null otherwise. */ + virtual std::shared_ptr getBuildLog(const Path & path) + { return nullptr; } + + /* Hack to allow long-running processes like hydra-queue-runner to + occasionally flush their path info cache. */ + void clearPathInfoCache() + { + state.lock()->pathInfoCache.clear(); + } + + /* Establish a connection to the store, for store types that have + a notion of connection. Otherwise this is a no-op. */ + virtual void connect() { }; + + /* Get the protocol version of this store or it's connection. */ + virtual unsigned int getProtocol() + { + return 0; + }; + + /* Get the priority of the store, used to order substituters. In + particular, binary caches can specify a priority field in their + "nix-cache-info" file. Lower value means higher priority. */ + virtual int getPriority() { return 0; } + + virtual Path toRealPath(const Path & storePath) + { + return storePath; + } + + virtual void createUser(const std::string & userName, uid_t userId) + { } + +protected: + + Stats stats; + + /* Unsupported methods. */ + [[noreturn]] void unsupported(const std::string & op) + { + throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri()); + } + +}; + + +class LocalFSStore : public virtual Store +{ +public: + + // FIXME: the (Store*) cast works around a bug in gcc that causes + // it to emit the call to the Option constructor. Clang works fine + // either way. + const PathSetting rootDir{(Store*) this, true, "", + "root", "directory prefixed to all other paths"}; + const PathSetting stateDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir, + "state", "directory where Nix will store state"}; + const PathSetting logDir{(Store*) this, false, + rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir, + "log", "directory where Nix will store state"}; + + const static string drvsLogDir; + + LocalFSStore(const Params & params); + + void narFromPath(const Path & path, Sink & sink) override; + ref getFSAccessor() override; + + /* Register a permanent GC root. */ + Path addPermRoot(const Path & storePath, + const Path & gcRoot, bool indirect, bool allowOutsideRootsDir = false); + + virtual Path getRealStoreDir() { return storeDir; } + + Path toRealPath(const Path & storePath) override + { + assert(isInStore(storePath)); + return getRealStoreDir() + "/" + std::string(storePath, storeDir.size() + 1); + } + + std::shared_ptr getBuildLog(const Path & path) override; +}; + + +/* Extract the name part of the given store path. */ +string storePathToName(const Path & path); + +/* Extract the hash part of the given store path. */ +string storePathToHash(const Path & path); + +/* Check whether ‘name’ is a valid store path name part, i.e. contains + only the characters [a-zA-Z0-9\+\-\.\_\?\=] and doesn't start with + a dot. */ +void checkStoreName(const string & name); + + +/* Copy a path from one store to another. */ +void copyStorePath(ref srcStore, ref dstStore, + const Path & storePath, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs); + + +/* Copy store paths from one store to another. The paths may be copied + in parallel. They are copied in a topologically sorted order + (i.e. if A is a reference of B, then A is copied before B), but + the set of store paths is not automatically closed; use + copyClosure() for that. */ +void copyPaths(ref srcStore, ref dstStore, const PathSet & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + + +/* Copy the closure of the specified paths from one store to another. */ +void copyClosure(ref srcStore, ref dstStore, + const PathSet & storePaths, + RepairFlag repair = NoRepair, + CheckSigsFlag checkSigs = CheckSigs, + SubstituteFlag substitute = NoSubstitute); + + +/* Remove the temporary roots file for this process. Any temporary + root becomes garbage after this point unless it has been registered + as a (permanent) root. */ +void removeTempRoots(); + + +/* Return a Store object to access the Nix store denoted by + ‘uri’ (slight misnomer...). Supported values are: + + * ‘local’: The Nix store in /nix/store and database in + /nix/var/nix/db, accessed directly. + + * ‘daemon’: The Nix store accessed via a Unix domain socket + connection to nix-daemon. + + * ‘unix://’: The Nix store accessed via a Unix domain socket + connection to nix-daemon, with the socket located at . + + * ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + whether the user has write access to the local Nix + store/database. + + * ‘file://’: A binary cache stored in . + + * ‘https://’: A binary cache accessed via HTTP. + + * ‘s3://’: A writable binary cache stored on Amazon's Simple + Storage Service. + + * ‘ssh://[user@]’: A remote Nix store accessed by running + ‘nix-store --serve’ via SSH. + + You can pass parameters to the store implementation by appending + ‘?key=value&key=value&...’ to the URI. +*/ +ref openStore(const std::string & uri = settings.storeUri.get(), + const Store::Params & extraParams = Store::Params()); + + +enum StoreType { + tDaemon, + tLocal, + tOther +}; + + +StoreType getStoreType(const std::string & uri = settings.storeUri.get(), + const std::string & stateDir = settings.nixStateDir); + +/* Return the default substituter stores, defined by the + ‘substituters’ option and various legacy options. */ +std::list> getDefaultSubstituters(); + + +/* Store implementation registration. */ +typedef std::function( + const std::string & uri, const Store::Params & params)> OpenStore; + +struct RegisterStoreImplementation +{ + typedef std::vector Implementations; + static Implementations * implementations; + + RegisterStoreImplementation(OpenStore fun) + { + if (!implementations) implementations = new Implementations; + implementations->push_back(fun); + } +}; + + + +/* Display a set of paths in human-readable form (i.e., between quotes + and separated by commas). */ +string showPaths(const PathSet & paths); + + +ValidPathInfo decodeValidPathInfo(std::istream & str, + bool hashGiven = false); + + +/* Compute the content-addressability assertion (ValidPathInfo::ca) + for paths created by makeFixedOutputPath() / addToStore(). */ +std::string makeFixedOutputCA(bool recursive, const Hash & hash); + + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri); + +} diff --git a/third_party/nix/src/libstore/worker-protocol.hh b/third_party/nix/src/libstore/worker-protocol.hh new file mode 100644 index 000000000..5ebdfaf13 --- /dev/null +++ b/third_party/nix/src/libstore/worker-protocol.hh @@ -0,0 +1,69 @@ +#pragma once + +namespace nix { + + +#define WORKER_MAGIC_1 0x6e697863 +#define WORKER_MAGIC_2 0x6478696f + +#define PROTOCOL_VERSION 0x115 +#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) +#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) + + +typedef enum { + wopIsValidPath = 1, + wopHasSubstitutes = 3, + wopQueryPathHash = 4, // obsolete + wopQueryReferences = 5, // obsolete + wopQueryReferrers = 6, + wopAddToStore = 7, + wopAddTextToStore = 8, + wopBuildPaths = 9, + wopEnsurePath = 10, + wopAddTempRoot = 11, + wopAddIndirectRoot = 12, + wopSyncWithGC = 13, + wopFindRoots = 14, + wopExportPath = 16, // obsolete + wopQueryDeriver = 18, // obsolete + wopSetOptions = 19, + wopCollectGarbage = 20, + wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, + wopQueryAllValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, + wopQueryPathInfo = 26, + wopImportPaths = 27, // obsolete + wopQueryDerivationOutputNames = 28, + wopQueryPathFromHashPart = 29, + wopQuerySubstitutablePathInfos = 30, + wopQueryValidPaths = 31, + wopQuerySubstitutablePaths = 32, + wopQueryValidDerivers = 33, + wopOptimiseStore = 34, + wopVerifyStore = 35, + wopBuildDerivation = 36, + wopAddSignatures = 37, + wopNarFromPath = 38, + wopAddToStoreNar = 39, + wopQueryMissing = 40, +} WorkerOp; + + +#define STDERR_NEXT 0x6f6c6d67 +#define STDERR_READ 0x64617461 // data needed from source +#define STDERR_WRITE 0x64617416 // data for sink +#define STDERR_LAST 0x616c7473 +#define STDERR_ERROR 0x63787470 +#define STDERR_START_ACTIVITY 0x53545254 +#define STDERR_STOP_ACTIVITY 0x53544f50 +#define STDERR_RESULT 0x52534c54 + + +Path readStorePath(Store & store, Source & from); +template T readStorePaths(Store & store, Source & from); + + +} diff --git a/third_party/nix/src/libutil/affinity.cc b/third_party/nix/src/libutil/affinity.cc new file mode 100644 index 000000000..98f8287ad --- /dev/null +++ b/third_party/nix/src/libutil/affinity.cc @@ -0,0 +1,55 @@ +#include "types.hh" +#include "util.hh" +#include "affinity.hh" + +#if __linux__ +#include +#endif + +namespace nix { + + +#if __linux__ +static bool didSaveAffinity = false; +static cpu_set_t savedAffinity; +#endif + + +void setAffinityTo(int cpu) +{ +#if __linux__ + if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; + didSaveAffinity = true; + debug(format("locking this thread to CPU %1%") % cpu); + cpu_set_t newAffinity; + CPU_ZERO(&newAffinity); + CPU_SET(cpu, &newAffinity); + if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) + printError(format("failed to lock thread to CPU %1%") % cpu); +#endif +} + + +int lockToCurrentCPU() +{ +#if __linux__ + int cpu = sched_getcpu(); + if (cpu != -1) setAffinityTo(cpu); + return cpu; +#else + return -1; +#endif +} + + +void restoreAffinity() +{ +#if __linux__ + if (!didSaveAffinity) return; + if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) + printError("failed to restore affinity %1%"); +#endif +} + + +} diff --git a/third_party/nix/src/libutil/affinity.hh b/third_party/nix/src/libutil/affinity.hh new file mode 100644 index 000000000..c1bd28e13 --- /dev/null +++ b/third_party/nix/src/libutil/affinity.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix { + +void setAffinityTo(int cpu); +int lockToCurrentCPU(); +void restoreAffinity(); + +} diff --git a/third_party/nix/src/libutil/archive.cc b/third_party/nix/src/libutil/archive.cc new file mode 100644 index 000000000..3aa120270 --- /dev/null +++ b/third_party/nix/src/libutil/archive.cc @@ -0,0 +1,378 @@ +#include +#include +#include +#include + +#include // for strcasecmp + +#include +#include +#include +#include +#include + +#include "archive.hh" +#include "util.hh" +#include "config.hh" + +namespace nix { + +struct ArchiveSettings : Config +{ + Setting useCaseHack{this, + #if __APPLE__ + true, + #else + false, + #endif + "use-case-hack", + "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; +}; + +static ArchiveSettings archiveSettings; + +static GlobalConfig::Register r1(&archiveSettings); + +const std::string narVersionMagic1 = "nix-archive-1"; + +static string caseHackSuffix = "~nix~case~hack~"; + +PathFilter defaultPathFilter = [](const Path &) { return true; }; + + +static void dumpContents(const Path & path, size_t size, + Sink & sink) +{ + sink << "contents" << size; + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); + + std::vector buf(65536); + size_t left = size; + + while (left > 0) { + auto n = std::min(left, buf.size()); + readFull(fd.get(), buf.data(), n); + left -= n; + sink(buf.data(), n); + } + + writePadding(size, sink); +} + + +static void dump(const Path & path, Sink & sink, PathFilter & filter) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting attributes of path '%1%'") % path); + + sink << "("; + + if (S_ISREG(st.st_mode)) { + sink << "type" << "regular"; + if (st.st_mode & S_IXUSR) + sink << "executable" << ""; + dumpContents(path, (size_t) st.st_size, sink); + } + + else if (S_ISDIR(st.st_mode)) { + sink << "type" << "directory"; + + /* If we're on a case-insensitive system like macOS, undo + the case hack applied by restorePath(). */ + std::map unhacked; + for (auto & i : readDirectory(path)) + if (archiveSettings.useCaseHack) { + string name(i.name); + size_t pos = i.name.find(caseHackSuffix); + if (pos != string::npos) { + debug(format("removing case hack suffix from '%1%'") % (path + "/" + i.name)); + name.erase(pos); + } + if (unhacked.find(name) != unhacked.end()) + throw Error(format("file name collision in between '%1%' and '%2%'") + % (path + "/" + unhacked[name]) % (path + "/" + i.name)); + unhacked[name] = i.name; + } else + unhacked[i.name] = i.name; + + for (auto & i : unhacked) + if (filter(path + "/" + i.first)) { + sink << "entry" << "(" << "name" << i.first << "node"; + dump(path + "/" + i.second, sink, filter); + sink << ")"; + } + } + + else if (S_ISLNK(st.st_mode)) + sink << "type" << "symlink" << "target" << readLink(path); + + else throw Error(format("file '%1%' has an unsupported type") % path); + + sink << ")"; +} + + +void dumpPath(const Path & path, Sink & sink, PathFilter & filter) +{ + sink << narVersionMagic1; + dump(path, sink, filter); +} + + +void dumpString(const std::string & s, Sink & sink) +{ + sink << narVersionMagic1 << "(" << "type" << "regular" << "contents" << s << ")"; +} + + +static SerialisationError badArchive(string s) +{ + return SerialisationError("bad archive: " + s); +} + + +#if 0 +static void skipGeneric(Source & source) +{ + if (readString(source) == "(") { + while (readString(source) != ")") + skipGeneric(source); + } +} +#endif + + +static void parseContents(ParseSink & sink, Source & source, const Path & path) +{ + unsigned long long size = readLongLong(source); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::vector buf(65536); + + while (left) { + checkInterrupt(); + auto n = buf.size(); + if ((unsigned long long)n > left) n = left; + source(buf.data(), n); + sink.receiveContents(buf.data(), n); + left -= n; + } + + readPadding(size, source); +} + + +struct CaseInsensitiveCompare +{ + bool operator() (const string & a, const string & b) const + { + return strcasecmp(a.c_str(), b.c_str()) < 0; + } +}; + + +static void parse(ParseSink & sink, Source & source, const Path & path) +{ + string s; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + enum { tpUnknown, tpRegular, tpDirectory, tpSymlink } type = tpUnknown; + + std::map names; + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } + + else if (s == "type") { + if (type != tpUnknown) + throw badArchive("multiple type fields"); + string t = readString(source); + + if (t == "regular") { + type = tpRegular; + sink.createRegularFile(path); + } + + else if (t == "directory") { + sink.createDirectory(path); + type = tpDirectory; + } + + else if (t == "symlink") { + type = tpSymlink; + } + + else throw badArchive("unknown file type " + t); + + } + + else if (s == "contents" && type == tpRegular) { + parseContents(sink, source, path); + } + + else if (s == "executable" && type == tpRegular) { + auto s = readString(source); + if (s != "") throw badArchive("executable marker has non-empty value"); + sink.isExecutable(); + } + + else if (s == "entry" && type == tpDirectory) { + string name, prevName; + + s = readString(source); + if (s != "(") throw badArchive("expected open tag"); + + while (1) { + checkInterrupt(); + + s = readString(source); + + if (s == ")") { + break; + } else if (s == "name") { + name = readString(source); + if (name.empty() || name == "." || name == ".." || name.find('/') != string::npos || name.find((char) 0) != string::npos) + throw Error(format("NAR contains invalid file name '%1%'") % name); + if (name <= prevName) + throw Error("NAR directory is not sorted"); + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + debug(format("case collision between '%1%' and '%2%'") % i->first % name); + name += caseHackSuffix; + name += std::to_string(++i->second); + } else + names[name] = 0; + } + } else if (s == "node") { + if (s.empty()) throw badArchive("entry name missing"); + parse(sink, source, path + "/" + name); + } else + throw badArchive("unknown field " + s); + } + } + + else if (s == "target" && type == tpSymlink) { + string target = readString(source); + sink.createSymlink(path, target); + } + + else + throw badArchive("unknown field " + s); + } +} + + +void parseDump(ParseSink & sink, Source & source) +{ + string version; + try { + version = readString(source, narVersionMagic1.size()); + } catch (SerialisationError & e) { + /* This generally means the integer at the start couldn't be + decoded. Ignore and throw the exception below. */ + } + if (version != narVersionMagic1) + throw badArchive("input doesn't look like a Nix archive"); + parse(sink, source, ""); +} + + +struct RestoreSink : ParseSink +{ + Path dstPath; + AutoCloseFD fd; + + void createDirectory(const Path & path) + { + Path p = dstPath + path; + if (mkdir(p.c_str(), 0777) == -1) + throw SysError(format("creating directory '%1%'") % p); + }; + + void createRegularFile(const Path & path) + { + Path p = dstPath + path; + fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666); + if (!fd) throw SysError(format("creating file '%1%'") % p); + } + + void isExecutable() + { + struct stat st; + if (fstat(fd.get(), &st) == -1) + throw SysError("fstat"); + if (fchmod(fd.get(), st.st_mode | (S_IXUSR | S_IXGRP | S_IXOTH)) == -1) + throw SysError("fchmod"); + } + + void preallocateContents(unsigned long long len) + { +#if HAVE_POSIX_FALLOCATE + if (len) { + errno = posix_fallocate(fd.get(), 0, len); + /* Note that EINVAL may indicate that the underlying + filesystem doesn't support preallocation (e.g. on + OpenSolaris). Since preallocation is just an + optimisation, ignore it. */ + if (errno && errno != EINVAL && errno != EOPNOTSUPP && errno != ENOSYS) + throw SysError(format("preallocating file of %1% bytes") % len); + } +#endif + } + + void receiveContents(unsigned char * data, unsigned int len) + { + writeFull(fd.get(), data, len); + } + + void createSymlink(const Path & path, const string & target) + { + Path p = dstPath + path; + nix::createSymlink(target, p); + } +}; + + +void restorePath(const Path & path, Source & source) +{ + RestoreSink sink; + sink.dstPath = path; + parseDump(sink, source); +} + + +void copyNAR(Source & source, Sink & sink) +{ + // FIXME: if 'source' is the output of dumpPath() followed by EOF, + // we should just forward all data directly without parsing. + + ParseSink parseSink; /* null sink; just parse the NAR */ + + LambdaSource wrapper([&](unsigned char * data, size_t len) { + auto n = source.read(data, len); + sink(data, n); + return n; + }); + + parseDump(parseSink, wrapper); +} + + +} diff --git a/third_party/nix/src/libutil/archive.hh b/third_party/nix/src/libutil/archive.hh new file mode 100644 index 000000000..25be426c1 --- /dev/null +++ b/third_party/nix/src/libutil/archive.hh @@ -0,0 +1,84 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +/* dumpPath creates a Nix archive of the specified path. The format + is as follows: + + IF path points to a REGULAR FILE: + dump(path) = attrs( + [ ("type", "regular") + , ("contents", contents(path)) + ]) + + IF path points to a DIRECTORY: + dump(path) = attrs( + [ ("type", "directory") + , ("entries", concat(map(f, sort(entries(path))))) + ]) + where f(fn) = attrs( + [ ("name", fn) + , ("file", dump(path + "/" + fn)) + ]) + + where: + + attrs(as) = concat(map(attr, as)) + encN(0) + attrs((a, b)) = encS(a) + encS(b) + + encS(s) = encN(len(s)) + s + (padding until next 64-bit boundary) + + encN(n) = 64-bit little-endian encoding of n. + + contents(path) = the contents of a regular file. + + sort(strings) = lexicographic sort by 8-bit value (strcmp). + + entries(path) = the entries of a directory, without `.' and + `..'. + + `+' denotes string concatenation. */ + + +void dumpPath(const Path & path, Sink & sink, + PathFilter & filter = defaultPathFilter); + +void dumpString(const std::string & s, Sink & sink); + +/* FIXME: fix this API, it sucks. */ +struct ParseSink +{ + virtual void createDirectory(const Path & path) { }; + + virtual void createRegularFile(const Path & path) { }; + virtual void isExecutable() { }; + virtual void preallocateContents(unsigned long long size) { }; + virtual void receiveContents(unsigned char * data, unsigned int len) { }; + + virtual void createSymlink(const Path & path, const string & target) { }; +}; + +struct TeeSink : ParseSink +{ + TeeSource source; + + TeeSink(Source & source) : source(source) { } +}; + +void parseDump(ParseSink & sink, Source & source); + +void restorePath(const Path & path, Source & source); + +/* Read a NAR from 'source' and write it to 'sink'. */ +void copyNAR(Source & source, Sink & sink); + + +extern const std::string narVersionMagic1; + + +} diff --git a/third_party/nix/src/libutil/args.cc b/third_party/nix/src/libutil/args.cc new file mode 100644 index 000000000..7af2a1bf7 --- /dev/null +++ b/third_party/nix/src/libutil/args.cc @@ -0,0 +1,203 @@ +#include "args.hh" +#include "hash.hh" + +namespace nix { + +Args::FlagMaker Args::mkFlag() +{ + return FlagMaker(*this); +} + +Args::FlagMaker::~FlagMaker() +{ + assert(flag->longName != ""); + args.longFlags[flag->longName] = flag; + if (flag->shortName) args.shortFlags[flag->shortName] = flag; +} + +void Args::parseCmdline(const Strings & _cmdline) +{ + Strings pendingArgs; + bool dashDash = false; + + Strings cmdline(_cmdline); + + for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { + + auto arg = *pos; + + /* Expand compound dash options (i.e., `-qlf' -> `-q -l -f', + `-j3` -> `-j 3`). */ + if (!dashDash && arg.length() > 2 && arg[0] == '-' && arg[1] != '-' && isalpha(arg[1])) { + *pos = (string) "-" + arg[1]; + auto next = pos; ++next; + for (unsigned int j = 2; j < arg.length(); j++) + if (isalpha(arg[j])) + cmdline.insert(next, (string) "-" + arg[j]); + else { + cmdline.insert(next, string(arg, j)); + break; + } + arg = *pos; + } + + if (!dashDash && arg == "--") { + dashDash = true; + ++pos; + } + else if (!dashDash && std::string(arg, 0, 1) == "-") { + if (!processFlag(pos, cmdline.end())) + throw UsageError(format("unrecognised flag '%1%'") % arg); + } + else { + pendingArgs.push_back(*pos++); + if (processArgs(pendingArgs, false)) + pendingArgs.clear(); + } + } + + processArgs(pendingArgs, true); +} + +void Args::printHelp(const string & programName, std::ostream & out) +{ + std::cout << "Usage: " << programName << " ..."; + for (auto & exp : expectedArgs) { + std::cout << renderLabels({exp.label}); + // FIXME: handle arity > 1 + if (exp.arity == 0) std::cout << "..."; + if (exp.optional) std::cout << "?"; + } + std::cout << "\n"; + + auto s = description(); + if (s != "") + std::cout << "\nSummary: " << s << ".\n"; + + if (longFlags.size()) { + std::cout << "\n"; + std::cout << "Flags:\n"; + printFlags(out); + } +} + +void Args::printFlags(std::ostream & out) +{ + Table2 table; + for (auto & flag : longFlags) { + if (hiddenCategories.count(flag.second->category)) continue; + table.push_back(std::make_pair( + (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ") + + "--" + flag.first + renderLabels(flag.second->labels), + flag.second->description)); + } + printTable(out, table); +} + +bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + assert(pos != end); + + auto process = [&](const std::string & name, const Flag & flag) -> bool { + ++pos; + std::vector args; + for (size_t n = 0 ; n < flag.arity; ++n) { + if (pos == end) { + if (flag.arity == ArityAny) break; + throw UsageError(format("flag '%1%' requires %2% argument(s)") + % name % flag.arity); + } + args.push_back(*pos++); + } + flag.handler(std::move(args)); + return true; + }; + + if (string(*pos, 0, 2) == "--") { + auto i = longFlags.find(string(*pos, 2)); + if (i == longFlags.end()) return false; + return process("--" + i->first, *i->second); + } + + if (string(*pos, 0, 1) == "-" && pos->size() == 2) { + auto c = (*pos)[1]; + auto i = shortFlags.find(c); + if (i == shortFlags.end()) return false; + return process(std::string("-") + c, *i->second); + } + + return false; +} + +bool Args::processArgs(const Strings & args, bool finish) +{ + if (expectedArgs.empty()) { + if (!args.empty()) + throw UsageError(format("unexpected argument '%1%'") % args.front()); + return true; + } + + auto & exp = expectedArgs.front(); + + bool res = false; + + if ((exp.arity == 0 && finish) || + (exp.arity > 0 && args.size() == exp.arity)) + { + std::vector ss; + for (auto & s : args) ss.push_back(s); + exp.handler(std::move(ss)); + expectedArgs.pop_front(); + res = true; + } + + if (finish && !expectedArgs.empty() && !expectedArgs.front().optional) + throw UsageError("more arguments are required"); + + return res; +} + +Args::FlagMaker & Args::FlagMaker::mkHashTypeFlag(HashType * ht) +{ + arity(1); + label("type"); + description("hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')"); + handler([ht](std::string s) { + *ht = parseHashType(s); + if (*ht == htUnknown) + throw UsageError("unknown hash type '%1%'", s); + }); + return *this; +} + +Strings argvToStrings(int argc, char * * argv) +{ + Strings args; + argc--; argv++; + while (argc--) args.push_back(*argv++); + return args; +} + +std::string renderLabels(const Strings & labels) +{ + std::string res; + for (auto label : labels) { + for (auto & c : label) c = std::toupper(c); + res += " <" + label + ">"; + } + return res; +} + +void printTable(std::ostream & out, const Table2 & table) +{ + size_t max = 0; + for (auto & row : table) + max = std::max(max, row.first.size()); + for (auto & row : table) { + out << " " << row.first + << std::string(max - row.first.size() + 2, ' ') + << row.second << "\n"; + } +} + +} diff --git a/third_party/nix/src/libutil/args.hh b/third_party/nix/src/libutil/args.hh new file mode 100644 index 000000000..ad5fcca39 --- /dev/null +++ b/third_party/nix/src/libutil/args.hh @@ -0,0 +1,201 @@ +#pragma once + +#include +#include +#include + +#include "util.hh" + +namespace nix { + +MakeError(UsageError, Error); + +enum HashType : char; + +class Args +{ +public: + + /* Parse the command line, throwing a UsageError if something goes + wrong. */ + void parseCmdline(const Strings & cmdline); + + virtual void printHelp(const string & programName, std::ostream & out); + + virtual std::string description() { return ""; } + +protected: + + static const size_t ArityAny = std::numeric_limits::max(); + + /* Flags. */ + struct Flag + { + typedef std::shared_ptr ptr; + std::string longName; + char shortName = 0; + std::string description; + Strings labels; + size_t arity = 0; + std::function)> handler; + std::string category; + }; + + std::map longFlags; + std::map shortFlags; + + virtual bool processFlag(Strings::iterator & pos, Strings::iterator end); + + virtual void printFlags(std::ostream & out); + + /* Positional arguments. */ + struct ExpectedArg + { + std::string label; + size_t arity; // 0 = any + bool optional; + std::function)> handler; + }; + + std::list expectedArgs; + + virtual bool processArgs(const Strings & args, bool finish); + + std::set hiddenCategories; + +public: + + class FlagMaker + { + Args & args; + Flag::ptr flag; + friend class Args; + FlagMaker(Args & args) : args(args), flag(std::make_shared()) { }; + public: + ~FlagMaker(); + FlagMaker & longName(const std::string & s) { flag->longName = s; return *this; }; + FlagMaker & shortName(char s) { flag->shortName = s; return *this; }; + FlagMaker & description(const std::string & s) { flag->description = s; return *this; }; + FlagMaker & label(const std::string & l) { flag->arity = 1; flag->labels = {l}; return *this; }; + FlagMaker & labels(const Strings & ls) { flag->arity = ls.size(); flag->labels = ls; return *this; }; + FlagMaker & arity(size_t arity) { flag->arity = arity; return *this; }; + FlagMaker & handler(std::function)> handler) { flag->handler = handler; return *this; }; + FlagMaker & handler(std::function handler) { flag->handler = [handler](std::vector) { handler(); }; return *this; }; + FlagMaker & handler(std::function handler) { + flag->arity = 1; + flag->handler = [handler](std::vector ss) { handler(std::move(ss[0])); }; + return *this; + }; + FlagMaker & category(const std::string & s) { flag->category = s; return *this; }; + + template + FlagMaker & dest(T * dest) + { + flag->arity = 1; + flag->handler = [=](std::vector ss) { *dest = ss[0]; }; + return *this; + }; + + template + FlagMaker & set(T * dest, const T & val) + { + flag->arity = 0; + flag->handler = [=](std::vector ss) { *dest = val; }; + return *this; + }; + + FlagMaker & mkHashTypeFlag(HashType * ht); + }; + + FlagMaker mkFlag(); + + /* Helper functions for constructing flags / positional + arguments. */ + + void mkFlag1(char shortName, const std::string & longName, + const std::string & label, const std::string & description, + std::function fun) + { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({label}) + .description(description) + .arity(1) + .handler([=](std::vector ss) { fun(ss[0]); }); + } + + void mkFlag(char shortName, const std::string & name, + const std::string & description, bool * dest) + { + mkFlag(shortName, name, description, dest, true); + } + + template + void mkFlag(char shortName, const std::string & longName, const std::string & description, + T * dest, const T & value) + { + mkFlag() + .shortName(shortName) + .longName(longName) + .description(description) + .handler([=](std::vector ss) { *dest = value; }); + } + + template + void mkIntFlag(char shortName, const std::string & longName, + const std::string & description, I * dest) + { + mkFlag(shortName, longName, description, [=](I n) { + *dest = n; + }); + } + + template + void mkFlag(char shortName, const std::string & longName, + const std::string & description, std::function fun) + { + mkFlag() + .shortName(shortName) + .longName(longName) + .labels({"N"}) + .description(description) + .arity(1) + .handler([=](std::vector ss) { + I n; + if (!string2Int(ss[0], n)) + throw UsageError("flag '--%s' requires a integer argument", longName); + fun(n); + }); + } + + /* Expect a string argument. */ + void expectArg(const std::string & label, string * dest, bool optional = false) + { + expectedArgs.push_back(ExpectedArg{label, 1, optional, [=](std::vector ss) { + *dest = ss[0]; + }}); + } + + /* Expect 0 or more arguments. */ + void expectArgs(const std::string & label, std::vector * dest) + { + expectedArgs.push_back(ExpectedArg{label, 0, false, [=](std::vector ss) { + *dest = std::move(ss); + }}); + } + + friend class MultiCommand; +}; + +Strings argvToStrings(int argc, char * * argv); + +/* Helper function for rendering argument labels. */ +std::string renderLabels(const Strings & labels); + +/* Helper function for printing 2-column tables. */ +typedef std::vector> Table2; + +void printTable(std::ostream & out, const Table2 & table); + +} diff --git a/third_party/nix/src/libutil/compression.cc b/third_party/nix/src/libutil/compression.cc new file mode 100644 index 000000000..0dd84e320 --- /dev/null +++ b/third_party/nix/src/libutil/compression.cc @@ -0,0 +1,432 @@ +#include "compression.hh" +#include "util.hh" +#include "finally.hh" +#include "logging.hh" + +#include +#include +#include +#include + +#include +#include + +#include + +namespace nix { + +// Don't feed brotli too much at once. +struct ChunkedCompressionSink : CompressionSink +{ + uint8_t outbuf[32 * 1024]; + + void write(const unsigned char * data, size_t len) override + { + const size_t CHUNK_SIZE = sizeof(outbuf) << 2; + while (len) { + size_t n = std::min(CHUNK_SIZE, len); + writeInternal(data, n); + data += n; + len -= n; + } + } + + virtual void writeInternal(const unsigned char * data, size_t len) = 0; +}; + +struct NoneSink : CompressionSink +{ + Sink & nextSink; + NoneSink(Sink & nextSink) : nextSink(nextSink) { } + void finish() override { flush(); } + void write(const unsigned char * data, size_t len) override { nextSink(data, len); } +}; + +struct XzDecompressionSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + lzma_ret ret = lzma_stream_decoder( + &strm, UINT64_MAX, LZMA_CONCATENATED); + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma decoder"); + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzDecompressionSink() + { + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char * data, size_t len) override + { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while decompressing xz file", ret); + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + bz_stream strm; + bool finished = false; + + BzipDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzDecompressInit(&strm, 0, 0); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 decoder"); + + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~BzipDecompressionSink() + { + BZ2_bzDecompressEnd(&strm); + } + + void finish() override + { + flush(); + write(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits::max()); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (strm.avail_in) { + checkInterrupt(); + + int ret = BZ2_bzDecompress(&strm); + if (ret != BZ_OK && ret != BZ_STREAM_END) + throw CompressionError("error while decompressing bzip2 file"); + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BrotliDecompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + BrotliDecoderState * state; + bool finished = false; + + BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) + { + state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialize brotli decoder"); + } + + ~BrotliDecompressionSink() + { + BrotliDecoderDestroyInstance(state); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliDecoderDecompressStream(state, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while decompressing brotli file"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliDecoderIsFinished(state); + } + } +}; + +ref decompress(const std::string & method, const std::string & in) +{ + StringSink ssink; + auto sink = makeDecompressionSink(method, ssink); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +ref makeDecompressionSink(const std::string & method, Sink & nextSink) +{ + if (method == "none" || method == "") + return make_ref(nextSink); + else if (method == "xz") + return make_ref(nextSink); + else if (method == "bzip2") + return make_ref(nextSink); + else if (method == "br") + return make_ref(nextSink); + else + throw UnknownCompressionMethod("unknown compression method '%s'", method); +} + +struct XzCompressionSink : CompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + lzma_stream strm = LZMA_STREAM_INIT; + bool finished = false; + + XzCompressionSink(Sink & nextSink, bool parallel) : nextSink(nextSink) + { + lzma_ret ret; + bool done = false; + + if (parallel) { +#ifdef HAVE_LZMA_MT + lzma_mt mt_options = {}; + mt_options.flags = 0; + mt_options.timeout = 300; // Using the same setting as the xz cmd line + mt_options.preset = LZMA_PRESET_DEFAULT; + mt_options.filters = NULL; + mt_options.check = LZMA_CHECK_CRC64; + mt_options.threads = lzma_cputhreads(); + mt_options.block_size = 0; + if (mt_options.threads == 0) + mt_options.threads = 1; + // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the + // number of threads. + ret = lzma_stream_encoder_mt(&strm, &mt_options); + done = true; +#else + printMsg(lvlError, "warning: parallel XZ compression requested but not supported, falling back to single-threaded compression"); +#endif + } + + if (!done) + ret = lzma_easy_encoder(&strm, 6, LZMA_CHECK_CRC64); + + if (ret != LZMA_OK) + throw CompressionError("unable to initialise lzma encoder"); + + // FIXME: apply the x86 BCJ filter? + + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~XzCompressionSink() + { + lzma_end(&strm); + } + + void finish() override + { + CompressionSink::flush(); + write(nullptr, 0); + } + + void write(const unsigned char * data, size_t len) override + { + strm.next_in = data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH); + if (ret != LZMA_OK && ret != LZMA_STREAM_END) + throw CompressionError("error %d while compressing xz file", ret); + + finished = ret == LZMA_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BzipCompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + bz_stream strm; + bool finished = false; + + BzipCompressionSink(Sink & nextSink) : nextSink(nextSink) + { + memset(&strm, 0, sizeof(strm)); + int ret = BZ2_bzCompressInit(&strm, 9, 0, 30); + if (ret != BZ_OK) + throw CompressionError("unable to initialise bzip2 encoder"); + + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + + ~BzipCompressionSink() + { + BZ2_bzCompressEnd(&strm); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + assert(len <= std::numeric_limits::max()); + + strm.next_in = (char *) data; + strm.avail_in = len; + + while (!finished && (!data || strm.avail_in)) { + checkInterrupt(); + + int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH); + if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + throw CompressionError("error %d while compressing bzip2 file", ret); + + finished = ret == BZ_STREAM_END; + + if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - strm.avail_out); + strm.next_out = (char *) outbuf; + strm.avail_out = sizeof(outbuf); + } + } + } +}; + +struct BrotliCompressionSink : ChunkedCompressionSink +{ + Sink & nextSink; + uint8_t outbuf[BUFSIZ]; + BrotliEncoderState *state; + bool finished = false; + + BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) + { + state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); + if (!state) + throw CompressionError("unable to initialise brotli encoder"); + } + + ~BrotliCompressionSink() + { + BrotliEncoderDestroyInstance(state); + } + + void finish() override + { + flush(); + writeInternal(nullptr, 0); + } + + void writeInternal(const unsigned char * data, size_t len) override + { + const uint8_t * next_in = data; + size_t avail_in = len; + uint8_t * next_out = outbuf; + size_t avail_out = sizeof(outbuf); + + while (!finished && (!data || avail_in)) { + checkInterrupt(); + + if (!BrotliEncoderCompressStream(state, + data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, &next_in, + &avail_out, &next_out, + nullptr)) + throw CompressionError("error while compressing brotli compression"); + + if (avail_out < sizeof(outbuf) || avail_in == 0) { + nextSink(outbuf, sizeof(outbuf) - avail_out); + next_out = outbuf; + avail_out = sizeof(outbuf); + } + + finished = BrotliEncoderIsFinished(state); + } + } +}; + +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel) +{ + if (method == "none") + return make_ref(nextSink); + else if (method == "xz") + return make_ref(nextSink, parallel); + else if (method == "bzip2") + return make_ref(nextSink); + else if (method == "br") + return make_ref(nextSink); + else + throw UnknownCompressionMethod(format("unknown compression method '%s'") % method); +} + +ref compress(const std::string & method, const std::string & in, const bool parallel) +{ + StringSink ssink; + auto sink = makeCompressionSink(method, ssink, parallel); + (*sink)(in); + sink->finish(); + return ssink.s; +} + +} diff --git a/third_party/nix/src/libutil/compression.hh b/third_party/nix/src/libutil/compression.hh new file mode 100644 index 000000000..dd666a4e1 --- /dev/null +++ b/third_party/nix/src/libutil/compression.hh @@ -0,0 +1,28 @@ +#pragma once + +#include "ref.hh" +#include "types.hh" +#include "serialise.hh" + +#include + +namespace nix { + +struct CompressionSink : BufferedSink +{ + virtual void finish() = 0; +}; + +ref decompress(const std::string & method, const std::string & in); + +ref makeDecompressionSink(const std::string & method, Sink & nextSink); + +ref compress(const std::string & method, const std::string & in, const bool parallel = false); + +ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false); + +MakeError(UnknownCompressionMethod, Error); + +MakeError(CompressionError, Error); + +} diff --git a/third_party/nix/src/libutil/config.cc b/third_party/nix/src/libutil/config.cc new file mode 100644 index 000000000..9023cb1bb --- /dev/null +++ b/third_party/nix/src/libutil/config.cc @@ -0,0 +1,338 @@ +#include "config.hh" +#include "args.hh" +#include "json.hh" + +namespace nix { + +bool Config::set(const std::string & name, const std::string & value) +{ + auto i = _settings.find(name); + if (i == _settings.end()) return false; + i->second.setting->set(value); + i->second.setting->overriden = true; + return true; +} + +void Config::addSetting(AbstractSetting * setting) +{ + _settings.emplace(setting->name, Config::SettingData(false, setting)); + for (auto & alias : setting->aliases) + _settings.emplace(alias, Config::SettingData(true, setting)); + + bool set = false; + + auto i = unknownSettings.find(setting->name); + if (i != unknownSettings.end()) { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + + for (auto & alias : setting->aliases) { + auto i = unknownSettings.find(alias); + if (i != unknownSettings.end()) { + if (set) + warn("setting '%s' is set, but it's an alias of '%s' which is also set", + alias, setting->name); + else { + setting->set(i->second); + setting->overriden = true; + unknownSettings.erase(i); + set = true; + } + } + } +} + +void AbstractConfig::warnUnknownSettings() +{ + for (auto & s : unknownSettings) + warn("unknown setting '%s'", s.first); +} + +void AbstractConfig::reapplyUnknownSettings() +{ + auto unknownSettings2 = std::move(unknownSettings); + for (auto & s : unknownSettings2) + set(s.first, s.second); +} + +void Config::getSettings(std::map & res, bool overridenOnly) +{ + for (auto & opt : _settings) + if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden)) + res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); +} + +void AbstractConfig::applyConfigFile(const Path & path) +{ + try { + string contents = readFile(path); + + unsigned int pos = 0; + + while (pos < contents.size()) { + string line; + while (pos < contents.size() && contents[pos] != '\n') + line += contents[pos++]; + pos++; + + string::size_type hash = line.find('#'); + if (hash != string::npos) + line = string(line, 0, hash); + + vector tokens = tokenizeString >(line); + if (tokens.empty()) continue; + + if (tokens.size() < 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + auto include = false; + auto ignoreMissing = false; + if (tokens[0] == "include") + include = true; + else if (tokens[0] == "!include") { + include = true; + ignoreMissing = true; + } + + if (include) { + if (tokens.size() != 2) + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + auto p = absPath(tokens[1], dirOf(path)); + if (pathExists(p)) { + applyConfigFile(p); + } else if (!ignoreMissing) { + throw Error("file '%1%' included from '%2%' not found", p, path); + } + continue; + } + + if (tokens[1] != "=") + throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + + string name = tokens[0]; + + vector::iterator i = tokens.begin(); + advance(i, 2); + + set(name, concatStringsSep(" ", Strings(i, tokens.end()))); // FIXME: slow + }; + } catch (SysError &) { } +} + +void Config::resetOverriden() +{ + for (auto & s : _settings) + s.second.setting->overriden = false; +} + +void Config::toJSON(JSONObject & out) +{ + for (auto & s : _settings) + if (!s.second.isAlias) { + JSONObject out2(out.object(s.first)); + out2.attr("description", s.second.setting->description); + JSONPlaceholder out3(out2.placeholder("value")); + s.second.setting->toJSON(out3); + } +} + +void Config::convertToArgs(Args & args, const std::string & category) +{ + for (auto & s : _settings) + if (!s.second.isAlias) + s.second.setting->convertToArg(args, category); +} + +AbstractSetting::AbstractSetting( + const std::string & name, + const std::string & description, + const std::set & aliases) + : name(name), description(description), aliases(aliases) +{ +} + +void AbstractSetting::toJSON(JSONPlaceholder & out) +{ + out.write(to_string()); +} + +void AbstractSetting::convertToArg(Args & args, const std::string & category) +{ +} + +template +void BaseSetting::toJSON(JSONPlaceholder & out) +{ + out.write(value); +} + +template +void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.mkFlag() + .longName(name) + .description(description) + .arity(1) + .handler([=](std::vector ss) { overriden = true; set(ss[0]); }) + .category(category); +} + +template<> void BaseSetting::set(const std::string & str) +{ + value = str; +} + +template<> std::string BaseSetting::to_string() +{ + return value; +} + +template +void BaseSetting::set(const std::string & str) +{ + static_assert(std::is_integral::value, "Integer required."); + if (!string2Int(str, value)) + throw UsageError("setting '%s' has invalid value '%s'", name, str); +} + +template +std::string BaseSetting::to_string() +{ + static_assert(std::is_integral::value, "Integer required."); + return std::to_string(value); +} + +template<> void BaseSetting::set(const std::string & str) +{ + if (str == "true" || str == "yes" || str == "1") + value = true; + else if (str == "false" || str == "no" || str == "0") + value = false; + else + throw UsageError("Boolean setting '%s' has invalid value '%s'", name, str); +} + +template<> std::string BaseSetting::to_string() +{ + return value ? "true" : "false"; +} + +template<> void BaseSetting::convertToArg(Args & args, const std::string & category) +{ + args.mkFlag() + .longName(name) + .description(description) + .handler([=](std::vector ss) { override(true); }) + .category(category); + args.mkFlag() + .longName("no-" + name) + .description(description) + .handler([=](std::vector ss) { override(false); }) + .category(category); +} + +template<> void BaseSetting::set(const std::string & str) +{ + value = tokenizeString(str); +} + +template<> std::string BaseSetting::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template<> void BaseSetting::set(const std::string & str) +{ + value = tokenizeString(str); +} + +template<> std::string BaseSetting::to_string() +{ + return concatStringsSep(" ", value); +} + +template<> void BaseSetting::toJSON(JSONPlaceholder & out) +{ + JSONList list(out.list()); + for (auto & s : value) + list.elem(s); +} + +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; +template class BaseSetting; + +void PathSetting::set(const std::string & str) +{ + if (str == "") { + if (allowEmpty) + value = ""; + else + throw UsageError("setting '%s' cannot be empty", name); + } else + value = canonPath(str); +} + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overridenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overridenOnly); +} + +void GlobalConfig::resetOverriden() +{ + for (auto & config : *configRegistrations) + config->resetOverriden(); +} + +void GlobalConfig::toJSON(JSONObject & out) +{ + for (auto & config : *configRegistrations) + config->toJSON(out); +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +} diff --git a/third_party/nix/src/libutil/config.hh b/third_party/nix/src/libutil/config.hh new file mode 100644 index 000000000..d86c65ff0 --- /dev/null +++ b/third_party/nix/src/libutil/config.hh @@ -0,0 +1,261 @@ +#include +#include + +#include "types.hh" + +#pragma once + +namespace nix { + +class Args; +class AbstractSetting; +class JSONPlaceholder; +class JSONObject; + +class AbstractConfig +{ +protected: + StringMap unknownSettings; + + AbstractConfig(const StringMap & initials = {}) + : unknownSettings(initials) + { } + +public: + + virtual bool set(const std::string & name, const std::string & value) = 0; + + struct SettingInfo + { + std::string value; + std::string description; + }; + + virtual void getSettings(std::map & res, bool overridenOnly = false) = 0; + + void applyConfigFile(const Path & path); + + virtual void resetOverriden() = 0; + + virtual void toJSON(JSONObject & out) = 0; + + virtual void convertToArgs(Args & args, const std::string & category) = 0; + + void warnUnknownSettings(); + + void reapplyUnknownSettings(); +}; + +/* A class to simplify providing configuration settings. The typical + use is to inherit Config and add Setting members: + + class MyClass : private Config + { + Setting foo{this, 123, "foo", "the number of foos to use"}; + Setting bar{this, "blabla", "bar", "the name of the bar"}; + + MyClass() : Config(readConfigFile("/etc/my-app.conf")) + { + std::cout << foo << "\n"; // will print 123 unless overriden + } + }; +*/ + +class Config : public AbstractConfig +{ + friend class AbstractSetting; + +public: + + struct SettingData + { + bool isAlias; + AbstractSetting * setting; + SettingData(bool isAlias, AbstractSetting * setting) + : isAlias(isAlias), setting(setting) + { } + }; + + typedef std::map Settings; + +private: + + Settings _settings; + +public: + + Config(const StringMap & initials = {}) + : AbstractConfig(initials) + { } + + bool set(const std::string & name, const std::string & value) override; + + void addSetting(AbstractSetting * setting); + + void getSettings(std::map & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; +}; + +class AbstractSetting +{ + friend class Config; + +public: + + const std::string name; + const std::string description; + const std::set aliases; + + int created = 123; + + bool overriden = false; + +protected: + + AbstractSetting( + const std::string & name, + const std::string & description, + const std::set & aliases); + + virtual ~AbstractSetting() + { + // Check against a gcc miscompilation causing our constructor + // not to run (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431). + assert(created == 123); + } + + virtual void set(const std::string & value) = 0; + + virtual std::string to_string() = 0; + + virtual void toJSON(JSONPlaceholder & out); + + virtual void convertToArg(Args & args, const std::string & category); + + bool isOverriden() { return overriden; } +}; + +/* A setting of type T. */ +template +class BaseSetting : public AbstractSetting +{ +protected: + + T value; + +public: + + BaseSetting(const T & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : AbstractSetting(name, description, aliases) + , value(def) + { } + + operator const T &() const { return value; } + operator T &() { return value; } + const T & get() const { return value; } + bool operator ==(const T & v2) const { return value == v2; } + bool operator !=(const T & v2) const { return value != v2; } + void operator =(const T & v) { assign(v); } + virtual void assign(const T & v) { value = v; } + + void set(const std::string & str) override; + + virtual void override(const T & v) + { + overriden = true; + value = v; + } + + std::string to_string() override; + + void convertToArg(Args & args, const std::string & category) override; + + void toJSON(JSONPlaceholder & out) override; +}; + +template +std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) +{ + str << (const T &) opt; + return str; +} + +template +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } + +template +class Setting : public BaseSetting +{ +public: + Setting(Config * options, + const T & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + { + options->addSetting(this); + } + + void operator =(const T & v) { this->assign(v); } +}; + +/* A special setting for Paths. These are automatically canonicalised + (e.g. "/foo//bar/" becomes "/foo/bar"). */ +class PathSetting : public BaseSetting +{ + bool allowEmpty; + +public: + + PathSetting(Config * options, + bool allowEmpty, + const Path & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, name, description, aliases) + , allowEmpty(allowEmpty) + { + options->addSetting(this); + } + + void set(const std::string & str) override; + + Path operator +(const char * p) const { return value + p; } + + void operator =(const Path & v) { this->assign(v); } +}; + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overridenOnly = false) override; + + void resetOverriden() override; + + void toJSON(JSONObject & out) override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/third_party/nix/src/libutil/finally.hh b/third_party/nix/src/libutil/finally.hh new file mode 100644 index 000000000..7760cfe9a --- /dev/null +++ b/third_party/nix/src/libutil/finally.hh @@ -0,0 +1,14 @@ +#pragma once + +#include + +/* A trivial class to run a function at the end of a scope. */ +class Finally +{ +private: + std::function fun; + +public: + Finally(std::function fun) : fun(fun) { } + ~Finally() { fun(); } +}; diff --git a/third_party/nix/src/libutil/hash.cc b/third_party/nix/src/libutil/hash.cc new file mode 100644 index 000000000..1c14ebb18 --- /dev/null +++ b/third_party/nix/src/libutil/hash.cc @@ -0,0 +1,355 @@ +#include +#include + +#include +#include + +#include "hash.hh" +#include "archive.hh" +#include "util.hh" +#include "istringstream_nocopy.hh" + +#include +#include +#include + +namespace nix { + + +void Hash::init() +{ + if (type == htMD5) hashSize = md5HashSize; + else if (type == htSHA1) hashSize = sha1HashSize; + else if (type == htSHA256) hashSize = sha256HashSize; + else if (type == htSHA512) hashSize = sha512HashSize; + else abort(); + assert(hashSize <= maxHashSize); + memset(hash, 0, maxHashSize); +} + + +bool Hash::operator == (const Hash & h2) const +{ + if (hashSize != h2.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) + if (hash[i] != h2.hash[i]) return false; + return true; +} + + +bool Hash::operator != (const Hash & h2) const +{ + return !(*this == h2); +} + + +bool Hash::operator < (const Hash & h) const +{ + if (hashSize < h.hashSize) return true; + if (hashSize > h.hashSize) return false; + for (unsigned int i = 0; i < hashSize; i++) { + if (hash[i] < h.hash[i]) return true; + if (hash[i] > h.hash[i]) return false; + } + return false; +} + + +const string base16Chars = "0123456789abcdef"; + + +static string printHash16(const Hash & hash) +{ + char buf[hash.hashSize * 2]; + for (unsigned int i = 0; i < hash.hashSize; i++) { + buf[i * 2] = base16Chars[hash.hash[i] >> 4]; + buf[i * 2 + 1] = base16Chars[hash.hash[i] & 0x0f]; + } + return string(buf, hash.hashSize * 2); +} + + +// omitted: E O U T +const string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; + + +static string printHash32(const Hash & hash) +{ + assert(hash.hashSize); + size_t len = hash.base32Len(); + assert(len); + + string s; + s.reserve(len); + + for (int n = (int) len - 1; n >= 0; n--) { + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + unsigned char c = + (hash.hash[i] >> j) + | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); + s.push_back(base32Chars[c & 0x1f]); + } + + return s; +} + + +string printHash16or32(const Hash & hash) +{ + return hash.to_string(hash.type == htMD5 ? Base16 : Base32, false); +} + + +std::string Hash::to_string(Base base, bool includeType) const +{ + std::string s; + if (base == SRI || includeType) { + s += printHashType(type); + s += base == SRI ? '-' : ':'; + } + switch (base) { + case Base16: + s += printHash16(*this); + break; + case Base32: + s += printHash32(*this); + break; + case Base64: + case SRI: + s += base64Encode(std::string((const char *) hash, hashSize)); + break; + } + return s; +} + + +Hash::Hash(const std::string & s, HashType type) + : type(type) +{ + size_t pos = 0; + bool isSRI = false; + + auto sep = s.find(':'); + if (sep == string::npos) { + sep = s.find('-'); + if (sep != string::npos) { + isSRI = true; + } else if (type == htUnknown) + throw BadHash("hash '%s' does not include a type", s); + } + + if (sep != string::npos) { + string hts = string(s, 0, sep); + this->type = parseHashType(hts); + if (this->type == htUnknown) + throw BadHash("unknown hash type '%s'", hts); + if (type != htUnknown && type != this->type) + throw BadHash("hash '%s' should have type '%s'", s, printHashType(type)); + pos = sep + 1; + } + + init(); + + size_t size = s.size() - pos; + + if (!isSRI && size == base16Len()) { + + auto parseHexDigit = [&](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + throw BadHash("invalid base-16 hash '%s'", s); + }; + + for (unsigned int i = 0; i < hashSize; i++) { + hash[i] = + parseHexDigit(s[pos + i * 2]) << 4 + | parseHexDigit(s[pos + i * 2 + 1]); + } + } + + else if (!isSRI && size == base32Len()) { + + for (unsigned int n = 0; n < size; ++n) { + char c = s[pos + size - n - 1]; + unsigned char digit; + for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ + if (base32Chars[digit] == c) break; + if (digit >= 32) + throw BadHash("invalid base-32 hash '%s'", s); + unsigned int b = n * 5; + unsigned int i = b / 8; + unsigned int j = b % 8; + hash[i] |= digit << j; + + if (i < hashSize - 1) { + hash[i + 1] |= digit >> (8 - j); + } else { + if (digit >> (8 - j)) + throw BadHash("invalid base-32 hash '%s'", s); + } + } + } + + else if (isSRI || size == base64Len()) { + auto d = base64Decode(std::string(s, pos)); + if (d.size() != hashSize) + throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", s); + assert(hashSize); + memcpy(hash, d.data(), hashSize); + } + + else + throw BadHash("hash '%s' has wrong length for hash type '%s'", s, printHashType(type)); +} + + +union Ctx +{ + MD5_CTX md5; + SHA_CTX sha1; + SHA256_CTX sha256; + SHA512_CTX sha512; +}; + + +static void start(HashType ht, Ctx & ctx) +{ + if (ht == htMD5) MD5_Init(&ctx.md5); + else if (ht == htSHA1) SHA1_Init(&ctx.sha1); + else if (ht == htSHA256) SHA256_Init(&ctx.sha256); + else if (ht == htSHA512) SHA512_Init(&ctx.sha512); +} + + +static void update(HashType ht, Ctx & ctx, + const unsigned char * bytes, size_t len) +{ + if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len); + else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len); + else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len); + else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len); +} + + +static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +{ + if (ht == htMD5) MD5_Final(hash, &ctx.md5); + else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); + else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); + else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); +} + + +Hash hashString(HashType ht, const string & s) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + update(ht, ctx, (const unsigned char *) s.data(), s.length()); + finish(ht, ctx, hash.hash); + return hash; +} + + +Hash hashFile(HashType ht, const Path & path) +{ + Ctx ctx; + Hash hash(ht); + start(ht, ctx); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError(format("opening file '%1%'") % path); + + std::vector buf(8192); + ssize_t n; + while ((n = read(fd.get(), buf.data(), buf.size()))) { + checkInterrupt(); + if (n == -1) throw SysError(format("reading file '%1%'") % path); + update(ht, ctx, buf.data(), n); + } + + finish(ht, ctx, hash.hash); + return hash; +} + + +HashSink::HashSink(HashType ht) : ht(ht) +{ + ctx = new Ctx; + bytes = 0; + start(ht, *ctx); +} + +HashSink::~HashSink() +{ + bufPos = 0; + delete ctx; +} + +void HashSink::write(const unsigned char * data, size_t len) +{ + bytes += len; + update(ht, *ctx, data, len); +} + +HashResult HashSink::finish() +{ + flush(); + Hash hash(ht); + nix::finish(ht, *ctx, hash.hash); + return HashResult(hash, bytes); +} + +HashResult HashSink::currentHash() +{ + flush(); + Ctx ctx2 = *ctx; + Hash hash(ht); + nix::finish(ht, ctx2, hash.hash); + return HashResult(hash, bytes); +} + + +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) +{ + HashSink sink(ht); + dumpPath(path, sink, filter); + return sink.finish(); +} + + +Hash compressHash(const Hash & hash, unsigned int newSize) +{ + Hash h; + h.hashSize = newSize; + for (unsigned int i = 0; i < hash.hashSize; ++i) + h.hash[i % newSize] ^= hash.hash[i]; + return h; +} + + +HashType parseHashType(const string & s) +{ + if (s == "md5") return htMD5; + else if (s == "sha1") return htSHA1; + else if (s == "sha256") return htSHA256; + else if (s == "sha512") return htSHA512; + else return htUnknown; +} + + +string printHashType(HashType ht) +{ + if (ht == htMD5) return "md5"; + else if (ht == htSHA1) return "sha1"; + else if (ht == htSHA256) return "sha256"; + else if (ht == htSHA512) return "sha512"; + else abort(); +} + + +} diff --git a/third_party/nix/src/libutil/hash.hh b/third_party/nix/src/libutil/hash.hh new file mode 100644 index 000000000..2dbc3b630 --- /dev/null +++ b/third_party/nix/src/libutil/hash.hh @@ -0,0 +1,131 @@ +#pragma once + +#include "types.hh" +#include "serialise.hh" + + +namespace nix { + + +MakeError(BadHash, Error); + + +enum HashType : char { htUnknown, htMD5, htSHA1, htSHA256, htSHA512 }; + + +const int md5HashSize = 16; +const int sha1HashSize = 20; +const int sha256HashSize = 32; +const int sha512HashSize = 64; + +extern const string base32Chars; + +enum Base : int { Base64, Base32, Base16, SRI }; + + +struct Hash +{ + static const unsigned int maxHashSize = 64; + unsigned int hashSize = 0; + unsigned char hash[maxHashSize] = {}; + + HashType type = htUnknown; + + /* Create an unset hash object. */ + Hash() { }; + + /* Create a zero-filled hash object. */ + Hash(HashType type) : type(type) { init(); }; + + /* Initialize the hash from a string representation, in the format + "[:]" or "-" (a + Subresource Integrity hash expression). If the 'type' argument + is htUnknown, then the hash type must be specified in the + string. */ + Hash(const std::string & s, HashType type = htUnknown); + + void init(); + + /* Check whether a hash is set. */ + operator bool () const { return type != htUnknown; } + + /* Check whether two hash are equal. */ + bool operator == (const Hash & h2) const; + + /* Check whether two hash are not equal. */ + bool operator != (const Hash & h2) const; + + /* For sorting. */ + bool operator < (const Hash & h) const; + + /* Returns the length of a base-16 representation of this hash. */ + size_t base16Len() const + { + return hashSize * 2; + } + + /* Returns the length of a base-32 representation of this hash. */ + size_t base32Len() const + { + return (hashSize * 8 - 1) / 5 + 1; + } + + /* Returns the length of a base-64 representation of this hash. */ + size_t base64Len() const + { + return ((4 * hashSize / 3) + 3) & ~3; + } + + /* Return a string representation of the hash, in base-16, base-32 + or base-64. By default, this is prefixed by the hash type + (e.g. "sha256:"). */ + std::string to_string(Base base = Base32, bool includeType = true) const; +}; + + +/* Print a hash in base-16 if it's MD5, or base-32 otherwise. */ +string printHash16or32(const Hash & hash); + +/* Compute the hash of the given string. */ +Hash hashString(HashType ht, const string & s); + +/* Compute the hash of the given file. */ +Hash hashFile(HashType ht, const Path & path); + +/* Compute the hash of the given path. The hash is defined as + (essentially) hashString(ht, dumpPath(path)). */ +typedef std::pair HashResult; +HashResult hashPath(HashType ht, const Path & path, + PathFilter & filter = defaultPathFilter); + +/* Compress a hash to the specified number of bytes by cyclically + XORing bytes together. */ +Hash compressHash(const Hash & hash, unsigned int newSize); + +/* Parse a string representing a hash type. */ +HashType parseHashType(const string & s); + +/* And the reverse. */ +string printHashType(HashType ht); + + +union Ctx; + +class HashSink : public BufferedSink +{ +private: + HashType ht; + Ctx * ctx; + unsigned long long bytes; + +public: + HashSink(HashType ht); + HashSink(const HashSink & h); + ~HashSink(); + void write(const unsigned char * data, size_t len); + HashResult finish(); + HashResult currentHash(); +}; + + +} diff --git a/third_party/nix/src/libutil/istringstream_nocopy.hh b/third_party/nix/src/libutil/istringstream_nocopy.hh new file mode 100644 index 000000000..f7beac578 --- /dev/null +++ b/third_party/nix/src/libutil/istringstream_nocopy.hh @@ -0,0 +1,92 @@ +/* This file provides a variant of std::istringstream that doesn't + copy its string argument. This is useful for large strings. The + caller must ensure that the string object is not destroyed while + it's referenced by this object. */ + +#pragma once + +#include +#include + +template , class Allocator = std::allocator> +class basic_istringbuf_nocopy : public std::basic_streambuf +{ +public: + typedef std::basic_string string_type; + + typedef typename std::basic_streambuf::off_type off_type; + + typedef typename std::basic_streambuf::pos_type pos_type; + + typedef typename std::basic_streambuf::int_type int_type; + + typedef typename std::basic_streambuf::traits_type traits_type; + +private: + const string_type & s; + + off_type off; + +public: + basic_istringbuf_nocopy(const string_type & s) : s{s}, off{0} + { + } + +private: + pos_type seekoff(off_type off, std::ios_base::seekdir dir, std::ios_base::openmode which) + { + if (which & std::ios_base::in) { + this->off = dir == std::ios_base::beg + ? off + : (dir == std::ios_base::end + ? s.size() + off + : this->off + off); + } + return pos_type(this->off); + } + + pos_type seekpos(pos_type pos, std::ios_base::openmode which) + { + return seekoff(pos, std::ios_base::beg, which); + } + + std::streamsize showmanyc() + { + return s.size() - off; + } + + int_type underflow() + { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off]); + } + + int_type uflow() + { + if (typename string_type::size_type(off) == s.size()) + return traits_type::eof(); + return traits_type::to_int_type(s[off++]); + } + + int_type pbackfail(int_type ch) + { + if (off == 0 || (ch != traits_type::eof() && ch != s[off - 1])) + return traits_type::eof(); + + return traits_type::to_int_type(s[--off]); + } + +}; + +template , class Allocator = std::allocator> +class basic_istringstream_nocopy : public std::basic_iostream +{ + typedef basic_istringbuf_nocopy buf_type; + buf_type buf; +public: + basic_istringstream_nocopy(const typename buf_type::string_type & s) : + std::basic_iostream(&buf), buf(s) {}; +}; + +typedef basic_istringstream_nocopy istringstream_nocopy; diff --git a/third_party/nix/src/libutil/json.cc b/third_party/nix/src/libutil/json.cc new file mode 100644 index 000000000..0a6fb65f0 --- /dev/null +++ b/third_party/nix/src/libutil/json.cc @@ -0,0 +1,174 @@ +#include "json.hh" + +#include +#include + +namespace nix { + +void toJSON(std::ostream & str, const char * start, const char * end) +{ + str << '"'; + for (auto i = start; i != end; i++) + if (*i == '\"' || *i == '\\') str << '\\' << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else if (*i >= 0 && *i < 32) + str << "\\u" << std::setfill('0') << std::setw(4) << std::hex << (uint16_t) *i << std::dec; + else str << *i; + str << '"'; +} + +void toJSON(std::ostream & str, const char * s) +{ + if (!s) str << "null"; else toJSON(str, s, s + strlen(s)); +} + +template<> void toJSON(std::ostream & str, const int & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned int & n) { str << n; } +template<> void toJSON(std::ostream & str, const long & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned long & n) { str << n; } +template<> void toJSON(std::ostream & str, const long long & n) { str << n; } +template<> void toJSON(std::ostream & str, const unsigned long long & n) { str << n; } +template<> void toJSON(std::ostream & str, const float & n) { str << n; } +template<> void toJSON(std::ostream & str, const double & n) { str << n; } + +template<> void toJSON(std::ostream & str, const std::string & s) +{ + toJSON(str, s.c_str(), s.c_str() + s.size()); +} + +template<> void toJSON(std::ostream & str, const bool & b) +{ + str << (b ? "true" : "false"); +} + +template<> void toJSON(std::ostream & str, const std::nullptr_t & b) +{ + str << "null"; +} + +JSONWriter::JSONWriter(std::ostream & str, bool indent) + : state(new JSONState(str, indent)) +{ + state->stack++; +} + +JSONWriter::JSONWriter(JSONState * state) + : state(state) +{ + state->stack++; +} + +JSONWriter::~JSONWriter() +{ + if (state) { + assertActive(); + state->stack--; + if (state->stack == 0) delete state; + } +} + +void JSONWriter::comma() +{ + assertActive(); + if (first) { + first = false; + } else { + state->str << ','; + } + if (state->indent) indent(); +} + +void JSONWriter::indent() +{ + state->str << '\n' << std::string(state->depth * 2, ' '); +} + +void JSONList::open() +{ + state->depth++; + state->str << '['; +} + +JSONList::~JSONList() +{ + state->depth--; + if (state->indent && !first) indent(); + state->str << "]"; +} + +JSONList JSONList::list() +{ + comma(); + return JSONList(state); +} + +JSONObject JSONList::object() +{ + comma(); + return JSONObject(state); +} + +JSONPlaceholder JSONList::placeholder() +{ + comma(); + return JSONPlaceholder(state); +} + +void JSONObject::open() +{ + state->depth++; + state->str << '{'; +} + +JSONObject::~JSONObject() +{ + if (state) { + state->depth--; + if (state->indent && !first) indent(); + state->str << "}"; + } +} + +void JSONObject::attr(const std::string & s) +{ + comma(); + toJSON(state->str, s); + state->str << ':'; + if (state->indent) state->str << ' '; +} + +JSONList JSONObject::list(const std::string & name) +{ + attr(name); + return JSONList(state); +} + +JSONObject JSONObject::object(const std::string & name) +{ + attr(name); + return JSONObject(state); +} + +JSONPlaceholder JSONObject::placeholder(const std::string & name) +{ + attr(name); + return JSONPlaceholder(state); +} + +JSONList JSONPlaceholder::list() +{ + assertValid(); + first = false; + return JSONList(state); +} + +JSONObject JSONPlaceholder::object() +{ + assertValid(); + first = false; + return JSONObject(state); +} + +} diff --git a/third_party/nix/src/libutil/json.hh b/third_party/nix/src/libutil/json.hh new file mode 100644 index 000000000..02a39917f --- /dev/null +++ b/third_party/nix/src/libutil/json.hh @@ -0,0 +1,189 @@ +#pragma once + +#include +#include +#include + +namespace nix { + +void toJSON(std::ostream & str, const char * start, const char * end); +void toJSON(std::ostream & str, const char * s); + +template +void toJSON(std::ostream & str, const T & n); + +class JSONWriter +{ +protected: + + struct JSONState + { + std::ostream & str; + bool indent; + size_t depth = 0; + size_t stack = 0; + JSONState(std::ostream & str, bool indent) : str(str), indent(indent) { } + ~JSONState() + { + assert(stack == 0); + } + }; + + JSONState * state; + + bool first = true; + + JSONWriter(std::ostream & str, bool indent); + + JSONWriter(JSONState * state); + + ~JSONWriter(); + + void assertActive() + { + assert(state->stack != 0); + } + + void comma(); + + void indent(); +}; + +class JSONObject; +class JSONPlaceholder; + +class JSONList : JSONWriter +{ +private: + + friend class JSONObject; + friend class JSONPlaceholder; + + void open(); + + JSONList(JSONState * state) + : JSONWriter(state) + { + open(); + } + +public: + + JSONList(std::ostream & str, bool indent = false) + : JSONWriter(str, indent) + { + open(); + } + + ~JSONList(); + + template + JSONList & elem(const T & v) + { + comma(); + toJSON(state->str, v); + return *this; + } + + JSONList list(); + + JSONObject object(); + + JSONPlaceholder placeholder(); +}; + +class JSONObject : JSONWriter +{ +private: + + friend class JSONList; + friend class JSONPlaceholder; + + void open(); + + JSONObject(JSONState * state) + : JSONWriter(state) + { + open(); + } + + void attr(const std::string & s); + +public: + + JSONObject(std::ostream & str, bool indent = false) + : JSONWriter(str, indent) + { + open(); + } + + JSONObject(const JSONObject & obj) = delete; + + JSONObject(JSONObject && obj) + : JSONWriter(obj.state) + { + obj.state = 0; + } + + ~JSONObject(); + + template + JSONObject & attr(const std::string & name, const T & v) + { + attr(name); + toJSON(state->str, v); + return *this; + } + + JSONList list(const std::string & name); + + JSONObject object(const std::string & name); + + JSONPlaceholder placeholder(const std::string & name); +}; + +class JSONPlaceholder : JSONWriter +{ + +private: + + friend class JSONList; + friend class JSONObject; + + JSONPlaceholder(JSONState * state) + : JSONWriter(state) + { + } + + void assertValid() + { + assertActive(); + assert(first); + } + +public: + + JSONPlaceholder(std::ostream & str, bool indent = false) + : JSONWriter(str, indent) + { + } + + ~JSONPlaceholder() + { + assert(!first || std::uncaught_exception()); + } + + template + void write(const T & v) + { + assertValid(); + first = false; + toJSON(state->str, v); + } + + JSONList list(); + + JSONObject object(); +}; + +} diff --git a/third_party/nix/src/libutil/lazy.hh b/third_party/nix/src/libutil/lazy.hh new file mode 100644 index 000000000..d073e486c --- /dev/null +++ b/third_party/nix/src/libutil/lazy.hh @@ -0,0 +1,48 @@ +#include +#include +#include + +namespace nix { + +/* A helper class for lazily-initialized variables. + + Lazy var([]() { return value; }); + + declares a variable of type T that is initialized to 'value' (in a + thread-safe way) on first use, that is, when var() is first + called. If the initialiser code throws an exception, then all + subsequent calls to var() will rethrow that exception. */ +template +class Lazy +{ + + typedef std::function Init; + + Init init; + + std::once_flag done; + + T value; + + std::exception_ptr ex; + +public: + + Lazy(Init init) : init(init) + { } + + const T & operator () () + { + std::call_once(done, [&]() { + try { + value = init(); + } catch (...) { + ex = std::current_exception(); + } + }); + if (ex) std::rethrow_exception(ex); + return value; + } +}; + +} diff --git a/third_party/nix/src/libutil/local.mk b/third_party/nix/src/libutil/local.mk new file mode 100644 index 000000000..e41a67d1f --- /dev/null +++ b/third_party/nix/src/libutil/local.mk @@ -0,0 +1,9 @@ +libraries += libutil + +libutil_NAME = libnixutil + +libutil_DIR := $(d) + +libutil_SOURCES := $(wildcard $(d)/*.cc) + +libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/third_party/nix/src/libutil/logging.cc b/third_party/nix/src/libutil/logging.cc new file mode 100644 index 000000000..b379306f6 --- /dev/null +++ b/third_party/nix/src/libutil/logging.cc @@ -0,0 +1,242 @@ +#include "logging.hh" +#include "util.hh" + +#include +#include + +namespace nix { + +static thread_local ActivityId curActivity = 0; + +ActivityId getCurActivity() +{ + return curActivity; +} +void setCurActivity(const ActivityId activityId) +{ + curActivity = activityId; +} + +Logger * logger = makeDefaultLogger(); + +void Logger::warn(const std::string & msg) +{ + log(lvlWarn, ANSI_RED "warning:" ANSI_NORMAL " " + msg); +} + +class SimpleLogger : public Logger +{ +public: + + bool systemd, tty; + + SimpleLogger() + { + systemd = getEnv("IN_SYSTEMD") == "1"; + tty = isatty(STDERR_FILENO); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + std::string prefix; + + if (systemd) { + char c; + switch (lvl) { + case lvlError: c = '3'; break; + case lvlWarn: c = '4'; break; + case lvlInfo: c = '5'; break; + case lvlTalkative: case lvlChatty: c = '6'; break; + default: c = '7'; + } + prefix = std::string("<") + c + ">"; + } + + writeToStderr(prefix + filterANSIEscapes(fs.s, !tty) + "\n"); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) + override + { + if (lvl <= verbosity && !s.empty()) + log(lvl, s + "..."); + } +}; + +Verbosity verbosity = lvlInfo; + +void warnOnce(bool & haveWarned, const FormatOrString & fs) +{ + if (!haveWarned) { + warn(fs.s); + haveWarned = true; + } +} + +void writeToStderr(const string & s) +{ + try { + writeFull(STDERR_FILENO, s, false); + } catch (SysError & e) { + /* Ignore failing writes to stderr. We need to ignore write + errors to ensure that cleanup code that logs to stderr runs + to completion if the other side of stderr has been closed + unexpectedly. */ + } +} + +Logger * makeDefaultLogger() +{ + return new SimpleLogger(); +} + +std::atomic nextId{(uint64_t) getpid() << 32}; + +Activity::Activity(Logger & logger, Verbosity lvl, ActivityType type, + const std::string & s, const Logger::Fields & fields, ActivityId parent) + : logger(logger), id(nextId++) +{ + logger.startActivity(id, lvl, type, s, fields, parent); +} + +struct JSONLogger : Logger +{ + Logger & prevLogger; + + JSONLogger(Logger & prevLogger) : prevLogger(prevLogger) { } + + void addFields(nlohmann::json & json, const Fields & fields) + { + if (fields.empty()) return; + auto & arr = json["fields"] = nlohmann::json::array(); + for (auto & f : fields) + if (f.type == Logger::Field::tInt) + arr.push_back(f.i); + else if (f.type == Logger::Field::tString) + arr.push_back(f.s); + else + abort(); + } + + void write(const nlohmann::json & json) + { + prevLogger.log(lvlError, "@nix " + json.dump()); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + nlohmann::json json; + json["action"] = "msg"; + json["level"] = lvl; + json["msg"] = fs.s; + write(json); + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + nlohmann::json json; + json["action"] = "start"; + json["id"] = act; + json["level"] = lvl; + json["type"] = type; + json["text"] = s; + addFields(json, fields); + // FIXME: handle parent + write(json); + } + + void stopActivity(ActivityId act) override + { + nlohmann::json json; + json["action"] = "stop"; + json["id"] = act; + write(json); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + addFields(json, fields); + write(json); + } +}; + +Logger * makeJSONLogger(Logger & prevLogger) +{ + return new JSONLogger(prevLogger); +} + +static Logger::Fields getFields(nlohmann::json & json) +{ + Logger::Fields fields; + for (auto & f : json) { + if (f.type() == nlohmann::json::value_t::number_unsigned) + fields.emplace_back(Logger::Field(f.get())); + else if (f.type() == nlohmann::json::value_t::string) + fields.emplace_back(Logger::Field(f.get())); + else throw Error("unsupported JSON type %d", (int) f.type()); + } + return fields; +} + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map & activities, bool trusted) +{ + if (!hasPrefix(msg, "@nix ")) return false; + + try { + auto json = nlohmann::json::parse(std::string(msg, 5)); + + std::string action = json["action"]; + + if (action == "start") { + auto type = (ActivityType) json["type"]; + if (trusted || type == actDownload) + activities.emplace(std::piecewise_construct, + std::forward_as_tuple(json["id"]), + std::forward_as_tuple(*logger, (Verbosity) json["level"], type, + json["text"], getFields(json["fields"]), act.id)); + } + + else if (action == "stop") + activities.erase((ActivityId) json["id"]); + + else if (action == "result") { + auto i = activities.find((ActivityId) json["id"]); + if (i != activities.end()) + i->second.result((ResultType) json["type"], getFields(json["fields"])); + } + + else if (action == "setPhase") { + std::string phase = json["phase"]; + act.result(resSetPhase, phase); + } + + else if (action == "msg") { + std::string msg = json["msg"]; + logger->log((Verbosity) json["level"], msg); + } + + } catch (std::exception & e) { + printError("bad log message from builder: %s", e.what()); + } + + return true; +} + +Activity::~Activity() { + try { + logger.stopActivity(id); + } catch (...) { + ignoreException(); + } +} + +} diff --git a/third_party/nix/src/libutil/logging.hh b/third_party/nix/src/libutil/logging.hh new file mode 100644 index 000000000..5df03da74 --- /dev/null +++ b/third_party/nix/src/libutil/logging.hh @@ -0,0 +1,172 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +typedef enum { + lvlError = 0, + lvlWarn, + lvlInfo, + lvlTalkative, + lvlChatty, + lvlDebug, + lvlVomit +} Verbosity; + +typedef enum { + actUnknown = 0, + actCopyPath = 100, + actDownload = 101, + actRealise = 102, + actCopyPaths = 103, + actBuilds = 104, + actBuild = 105, + actOptimiseStore = 106, + actVerifyPaths = 107, + actSubstitute = 108, + actQueryPathInfo = 109, + actPostBuildHook = 110, +} ActivityType; + +typedef enum { + resFileLinked = 100, + resBuildLogLine = 101, + resUntrustedPath = 102, + resCorruptedPath = 103, + resSetPhase = 104, + resProgress = 105, + resSetExpected = 106, + resPostBuildLogLine = 107, +} ResultType; + +typedef uint64_t ActivityId; + +class Logger +{ + friend struct Activity; + +public: + + struct Field + { + // FIXME: use std::variant. + enum { tInt = 0, tString = 1 } type; + uint64_t i = 0; + std::string s; + Field(const std::string & s) : type(tString), s(s) { } + Field(const char * s) : type(tString), s(s) { } + Field(const uint64_t & i) : type(tInt), i(i) { } + }; + + typedef std::vector Fields; + + virtual ~Logger() { } + + virtual void log(Verbosity lvl, const FormatOrString & fs) = 0; + + void log(const FormatOrString & fs) + { + log(lvlInfo, fs); + } + + virtual void warn(const std::string & msg); + + virtual void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) { }; + + virtual void stopActivity(ActivityId act) { }; + + virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; +}; + +ActivityId getCurActivity(); +void setCurActivity(const ActivityId activityId); + +struct Activity +{ + Logger & logger; + + const ActivityId id; + + Activity(Logger & logger, Verbosity lvl, ActivityType type, const std::string & s = "", + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()); + + Activity(Logger & logger, ActivityType type, + const Logger::Fields & fields = {}, ActivityId parent = getCurActivity()) + : Activity(logger, lvlError, type, "", fields, parent) { }; + + Activity(const Activity & act) = delete; + + ~Activity(); + + void progress(uint64_t done = 0, uint64_t expected = 0, uint64_t running = 0, uint64_t failed = 0) const + { result(resProgress, done, expected, running, failed); } + + void setExpected(ActivityType type2, uint64_t expected) const + { result(resSetExpected, type2, expected); } + + template + void result(ResultType type, const Args & ... args) const + { + Logger::Fields fields; + nop{(fields.emplace_back(Logger::Field(args)), 1)...}; + result(type, fields); + } + + void result(ResultType type, const Logger::Fields & fields) const + { + logger.result(id, type, fields); + } + + friend class Logger; +}; + +struct PushActivity +{ + const ActivityId prevAct; + PushActivity(ActivityId act) : prevAct(getCurActivity()) { setCurActivity(act); } + ~PushActivity() { setCurActivity(prevAct); } +}; + +extern Logger * logger; + +Logger * makeDefaultLogger(); + +Logger * makeJSONLogger(Logger & prevLogger); + +bool handleJSONLogMessage(const std::string & msg, + const Activity & act, std::map & activities, + bool trusted); + +extern Verbosity verbosity; /* suppress msgs > this */ + +/* Print a message if the current log level is at least the specified + level. Note that this has to be implemented as a macro to ensure + that the arguments are evaluated lazily. */ +#define printMsg(level, args...) \ + do { \ + if (level <= nix::verbosity) { \ + logger->log(level, fmt(args)); \ + } \ + } while (0) + +#define printError(args...) printMsg(lvlError, args) +#define printInfo(args...) printMsg(lvlInfo, args) +#define printTalkative(args...) printMsg(lvlTalkative, args) +#define debug(args...) printMsg(lvlDebug, args) +#define vomit(args...) printMsg(lvlVomit, args) + +template +inline void warn(const std::string & fs, Args... args) +{ + boost::format f(fs); + nop{boost::io::detail::feed(f, args)...}; + logger->warn(f.str()); +} + +void warnOnce(bool & haveWarned, const FormatOrString & fs); + +void writeToStderr(const string & s); + +} diff --git a/third_party/nix/src/libutil/lru-cache.hh b/third_party/nix/src/libutil/lru-cache.hh new file mode 100644 index 000000000..8b83f842c --- /dev/null +++ b/third_party/nix/src/libutil/lru-cache.hh @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include + +namespace nix { + +/* A simple least-recently used cache. Not thread-safe. */ +template +class LRUCache +{ +private: + + size_t capacity; + + // Stupid wrapper to get around circular dependency between Data + // and LRU. + struct LRUIterator; + + using Data = std::map>; + using LRU = std::list; + + struct LRUIterator { typename LRU::iterator it; }; + + Data data; + LRU lru; + +public: + + LRUCache(size_t capacity) : capacity(capacity) { } + + /* Insert or upsert an item in the cache. */ + void upsert(const Key & key, const Value & value) + { + if (capacity == 0) return; + + erase(key); + + if (data.size() >= capacity) { + /* Retire the oldest item. */ + auto oldest = lru.begin(); + data.erase(*oldest); + lru.erase(oldest); + } + + auto res = data.emplace(key, std::make_pair(LRUIterator(), value)); + assert(res.second); + auto & i(res.first); + + auto j = lru.insert(lru.end(), i); + + i->second.first.it = j; + } + + bool erase(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return false; + lru.erase(i->second.first.it); + data.erase(i); + return true; + } + + /* Look up an item in the cache. If it exists, it becomes the most + recently used item. */ + std::optional get(const Key & key) + { + auto i = data.find(key); + if (i == data.end()) return {}; + + /* Move this item to the back of the LRU list. */ + lru.erase(i->second.first.it); + auto j = lru.insert(lru.end(), i); + i->second.first.it = j; + + return i->second.second; + } + + size_t size() + { + return data.size(); + } + + void clear() + { + data.clear(); + lru.clear(); + } +}; + +} diff --git a/third_party/nix/src/libutil/monitor-fd.hh b/third_party/nix/src/libutil/monitor-fd.hh new file mode 100644 index 000000000..5ee0b88ef --- /dev/null +++ b/third_party/nix/src/libutil/monitor-fd.hh @@ -0,0 +1,58 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace nix { + + +class MonitorFdHup +{ +private: + std::thread thread; + +public: + MonitorFdHup(int fd) + { + thread = std::thread([fd]() { + while (true) { + /* Wait indefinitely until a POLLHUP occurs. */ + struct pollfd fds[1]; + fds[0].fd = fd; + /* This shouldn't be necessary, but macOS doesn't seem to + like a zeroed out events field. + See rdar://37537852. + */ + fds[0].events = POLLHUP; + auto count = poll(fds, 1, -1); + if (count == -1) abort(); // can't happen + /* This shouldn't happen, but can on macOS due to a bug. + See rdar://37550628. + + This may eventually need a delay or further + coordination with the main thread if spinning proves + too harmful. + */ + if (count == 0) continue; + assert(fds[0].revents & POLLHUP); + triggerInterrupt(); + break; + } + }); + }; + + ~MonitorFdHup() + { + pthread_cancel(thread.native_handle()); + thread.join(); + } +}; + + +} diff --git a/third_party/nix/src/libutil/pool.hh b/third_party/nix/src/libutil/pool.hh new file mode 100644 index 000000000..d49067bb9 --- /dev/null +++ b/third_party/nix/src/libutil/pool.hh @@ -0,0 +1,187 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "sync.hh" +#include "ref.hh" + +namespace nix { + +/* This template class implements a simple pool manager of resources + of some type R, such as database connections. It is used as + follows: + + class Connection { ... }; + + Pool pool; + + { + auto conn(pool.get()); + conn->exec("select ..."); + } + + Here, the Connection object referenced by ‘conn’ is automatically + returned to the pool when ‘conn’ goes out of scope. +*/ + +template +class Pool +{ +public: + + /* A function that produces new instances of R on demand. */ + typedef std::function()> Factory; + + /* A function that checks whether an instance of R is still + usable. Unusable instances are removed from the pool. */ + typedef std::function &)> Validator; + +private: + + Factory factory; + Validator validator; + + struct State + { + size_t inUse = 0; + size_t max; + std::vector> idle; + }; + + Sync state; + + std::condition_variable wakeup; + +public: + + Pool(size_t max = std::numeric_limits::max(), + const Factory & factory = []() { return make_ref(); }, + const Validator & validator = [](ref r) { return true; }) + : factory(factory) + , validator(validator) + { + auto state_(state.lock()); + state_->max = max; + } + + void incCapacity() + { + auto state_(state.lock()); + state_->max++; + /* we could wakeup here, but this is only used when we're + * about to nest Pool usages, and we want to save the slot for + * the nested use if we can + */ + } + + void decCapacity() + { + auto state_(state.lock()); + state_->max--; + } + + ~Pool() + { + auto state_(state.lock()); + assert(!state_->inUse); + state_->max = 0; + state_->idle.clear(); + } + + class Handle + { + private: + Pool & pool; + std::shared_ptr r; + bool bad = false; + + friend Pool; + + Handle(Pool & pool, std::shared_ptr r) : pool(pool), r(r) { } + + public: + Handle(Handle && h) : pool(h.pool), r(h.r) { h.r.reset(); } + + Handle(const Handle & l) = delete; + + ~Handle() + { + if (!r) return; + { + auto state_(pool.state.lock()); + if (!bad) + state_->idle.push_back(ref(r)); + assert(state_->inUse); + state_->inUse--; + } + pool.wakeup.notify_one(); + } + + R * operator -> () { return &*r; } + R & operator * () { return *r; } + + void markBad() { bad = true; } + }; + + Handle get() + { + { + auto state_(state.lock()); + + /* If we're over the maximum number of instance, we need + to wait until a slot becomes available. */ + while (state_->idle.empty() && state_->inUse >= state_->max) + state_.wait(wakeup); + + while (!state_->idle.empty()) { + auto p = state_->idle.back(); + state_->idle.pop_back(); + if (validator(p)) { + state_->inUse++; + return Handle(*this, p); + } + } + + state_->inUse++; + } + + /* We need to create a new instance. Because that might take a + while, we don't hold the lock in the meantime. */ + try { + Handle h(*this, factory()); + return h; + } catch (...) { + auto state_(state.lock()); + state_->inUse--; + wakeup.notify_one(); + throw; + } + } + + size_t count() + { + auto state_(state.lock()); + return state_->idle.size() + state_->inUse; + } + + size_t capacity() + { + return state.lock()->max; + } + + void flushBad() + { + auto state_(state.lock()); + std::vector> left; + for (auto & p : state_->idle) + if (validator(p)) + left.push_back(p); + std::swap(state_->idle, left); + } +}; + +} diff --git a/third_party/nix/src/libutil/ref.hh b/third_party/nix/src/libutil/ref.hh new file mode 100644 index 000000000..0be2a7e74 --- /dev/null +++ b/third_party/nix/src/libutil/ref.hh @@ -0,0 +1,92 @@ +#pragma once + +#include +#include +#include + +namespace nix { + +/* A simple non-nullable reference-counted pointer. Actually a wrapper + around std::shared_ptr that prevents non-null constructions. */ +template +class ref +{ +private: + + std::shared_ptr p; + +public: + + ref(const ref & r) + : p(r.p) + { } + + explicit ref(const std::shared_ptr & p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + explicit ref(T * p) + : p(p) + { + if (!p) + throw std::invalid_argument("null pointer cast to ref"); + } + + T* operator ->() const + { + return &*p; + } + + T& operator *() const + { + return *p; + } + + operator std::shared_ptr () const + { + return p; + } + + std::shared_ptr get_ptr() const + { + return p; + } + + template + ref cast() const + { + return ref(std::dynamic_pointer_cast(p)); + } + + template + std::shared_ptr dynamic_pointer_cast() const + { + return std::dynamic_pointer_cast(p); + } + + template + operator ref () const + { + return ref((std::shared_ptr) p); + } + +private: + + template + friend ref + make_ref(Args&&... args); + +}; + +template +inline ref +make_ref(Args&&... args) +{ + auto p = std::make_shared(std::forward(args)...); + return ref(p); +} + +} diff --git a/third_party/nix/src/libutil/serialise.cc b/third_party/nix/src/libutil/serialise.cc new file mode 100644 index 000000000..8201549fd --- /dev/null +++ b/third_party/nix/src/libutil/serialise.cc @@ -0,0 +1,323 @@ +#include "serialise.hh" +#include "util.hh" + +#include +#include +#include + +#include + + +namespace nix { + + +void BufferedSink::operator () (const unsigned char * data, size_t len) +{ + if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + + while (len) { + /* Optimisation: bypass the buffer if the data exceeds the + buffer size. */ + if (bufPos + len >= bufSize) { + flush(); + write(data, len); + break; + } + /* Otherwise, copy the bytes to the buffer. Flush the buffer + when it's full. */ + size_t n = bufPos + len > bufSize ? bufSize - bufPos : len; + memcpy(buffer.get() + bufPos, data, n); + data += n; bufPos += n; len -= n; + if (bufPos == bufSize) flush(); + } +} + + +void BufferedSink::flush() +{ + if (bufPos == 0) return; + size_t n = bufPos; + bufPos = 0; // don't trigger the assert() in ~BufferedSink() + write(buffer.get(), n); +} + + +FdSink::~FdSink() +{ + try { flush(); } catch (...) { ignoreException(); } +} + + +size_t threshold = 256 * 1024 * 1024; + +static void warnLargeDump() +{ + printError("warning: dumping very large path (> 256 MiB); this may run out of memory"); +} + + +void FdSink::write(const unsigned char * data, size_t len) +{ + written += len; + static bool warned = false; + if (warn && !warned) { + if (written > threshold) { + warnLargeDump(); + warned = true; + } + } + try { + writeFull(fd, data, len); + } catch (SysError & e) { + _good = false; + throw; + } +} + + +bool FdSink::good() +{ + return _good; +} + + +void Source::operator () (unsigned char * data, size_t len) +{ + while (len) { + size_t n = read(data, len); + data += n; len -= n; + } +} + + +std::string Source::drain() +{ + std::string s; + std::vector buf(8192); + while (true) { + size_t n; + try { + n = read(buf.data(), buf.size()); + s.append((char *) buf.data(), n); + } catch (EndOfFile &) { + break; + } + } + return s; +} + + +size_t BufferedSource::read(unsigned char * data, size_t len) +{ + if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]); + + if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize); + + /* Copy out the data in the buffer. */ + size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + memcpy(data, buffer.get() + bufPosOut, n); + bufPosOut += n; + if (bufPosIn == bufPosOut) bufPosIn = bufPosOut = 0; + return n; +} + + +bool BufferedSource::hasData() +{ + return bufPosOut < bufPosIn; +} + + +size_t FdSource::readUnbuffered(unsigned char * data, size_t len) +{ + ssize_t n; + do { + checkInterrupt(); + n = ::read(fd, (char *) data, len); + } while (n == -1 && errno == EINTR); + if (n == -1) { _good = false; throw SysError("reading from file"); } + if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); } + read += n; + return n; +} + + +bool FdSource::good() +{ + return _good; +} + + +size_t StringSource::read(unsigned char * data, size_t len) +{ + if (pos == s.size()) throw EndOfFile("end of string reached"); + size_t n = s.copy((char *) data, len, pos); + pos += n; + return n; +} + + +#if BOOST_VERSION >= 106300 && BOOST_VERSION < 106600 +#error Coroutines are broken in this version of Boost! +#endif + +std::unique_ptr sinkToSource( + std::function fun, + std::function eof) +{ + struct SinkToSource : Source + { + typedef boost::coroutines2::coroutine coro_t; + + std::function fun; + std::function eof; + std::optional coro; + bool started = false; + + SinkToSource(std::function fun, std::function eof) + : fun(fun), eof(eof) + { + } + + std::string cur; + size_t pos = 0; + + size_t read(unsigned char * data, size_t len) override + { + if (!coro) + coro = coro_t::pull_type([&](coro_t::push_type & yield) { + LambdaSink sink([&](const unsigned char * data, size_t len) { + if (len) yield(std::string((const char *) data, len)); + }); + fun(sink); + }); + + if (!*coro) { eof(); abort(); } + + if (pos == cur.size()) { + if (!cur.empty()) (*coro)(); + cur = coro->get(); + pos = 0; + } + + auto n = std::min(cur.size() - pos, len); + memcpy(data, (unsigned char *) cur.data() + pos, n); + pos += n; + + return n; + } + }; + + return std::make_unique(fun, eof); +} + + +void writePadding(size_t len, Sink & sink) +{ + if (len % 8) { + unsigned char zero[8]; + memset(zero, 0, sizeof(zero)); + sink(zero, 8 - (len % 8)); + } +} + + +void writeString(const unsigned char * buf, size_t len, Sink & sink) +{ + sink << len; + sink(buf, len); + writePadding(len, sink); +} + + +Sink & operator << (Sink & sink, const string & s) +{ + writeString((const unsigned char *) s.data(), s.size(), sink); + return sink; +} + + +template void writeStrings(const T & ss, Sink & sink) +{ + sink << ss.size(); + for (auto & i : ss) + sink << i; +} + +Sink & operator << (Sink & sink, const Strings & s) +{ + writeStrings(s, sink); + return sink; +} + +Sink & operator << (Sink & sink, const StringSet & s) +{ + writeStrings(s, sink); + return sink; +} + + +void readPadding(size_t len, Source & source) +{ + if (len % 8) { + unsigned char zero[8]; + size_t n = 8 - (len % 8); + source(zero, n); + for (unsigned int i = 0; i < n; i++) + if (zero[i]) throw SerialisationError("non-zero padding"); + } +} + + +size_t readString(unsigned char * buf, size_t max, Source & source) +{ + auto len = readNum(source); + if (len > max) throw SerialisationError("string is too long"); + source(buf, len); + readPadding(len, source); + return len; +} + + +string readString(Source & source, size_t max) +{ + auto len = readNum(source); + if (len > max) throw SerialisationError("string is too long"); + std::string res(len, 0); + source((unsigned char*) res.data(), len); + readPadding(len, source); + return res; +} + +Source & operator >> (Source & in, string & s) +{ + s = readString(in); + return in; +} + + +template T readStrings(Source & source) +{ + auto count = readNum(source); + T ss; + while (count--) + ss.insert(ss.end(), readString(source)); + return ss; +} + +template Paths readStrings(Source & source); +template PathSet readStrings(Source & source); + + +void StringSink::operator () (const unsigned char * data, size_t len) +{ + static bool warned = false; + if (!warned && s->size() > threshold) { + warnLargeDump(); + warned = true; + } + s->append((const char *) data, len); +} + + +} diff --git a/third_party/nix/src/libutil/serialise.hh b/third_party/nix/src/libutil/serialise.hh new file mode 100644 index 000000000..a344a5ac7 --- /dev/null +++ b/third_party/nix/src/libutil/serialise.hh @@ -0,0 +1,337 @@ +#pragma once + +#include + +#include "types.hh" +#include "util.hh" + + +namespace nix { + + +/* Abstract destination of binary data. */ +struct Sink +{ + virtual ~Sink() { } + virtual void operator () (const unsigned char * data, size_t len) = 0; + virtual bool good() { return true; } + + void operator () (const std::string & s) + { + (*this)((const unsigned char *) s.data(), s.size()); + } +}; + + +/* A buffered abstract sink. */ +struct BufferedSink : Sink +{ + size_t bufSize, bufPos; + std::unique_ptr buffer; + + BufferedSink(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPos(0), buffer(nullptr) { } + + void operator () (const unsigned char * data, size_t len) override; + + void operator () (const std::string & s) + { + Sink::operator()(s); + } + + void flush(); + + virtual void write(const unsigned char * data, size_t len) = 0; +}; + + +/* Abstract source of binary data. */ +struct Source +{ + virtual ~Source() { } + + /* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’. + It blocks until all the requested data is available, or throws + an error if it is not going to be available. */ + void operator () (unsigned char * data, size_t len); + + /* Store up to ‘len’ in the buffer pointed to by ‘data’, and + return the number of bytes stored. It blocks until at least + one byte is available. */ + virtual size_t read(unsigned char * data, size_t len) = 0; + + virtual bool good() { return true; } + + std::string drain(); +}; + + +/* A buffered abstract source. */ +struct BufferedSource : Source +{ + size_t bufSize, bufPosIn, bufPosOut; + std::unique_ptr buffer; + + BufferedSource(size_t bufSize = 32 * 1024) + : bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { } + + size_t read(unsigned char * data, size_t len) override; + + + bool hasData(); + +protected: + /* Underlying read call, to be overridden. */ + virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0; +}; + + +/* A sink that writes data to a file descriptor. */ +struct FdSink : BufferedSink +{ + int fd; + bool warn = false; + size_t written = 0; + + FdSink() : fd(-1) { } + FdSink(int fd) : fd(fd) { } + FdSink(FdSink&&) = default; + + FdSink& operator=(FdSink && s) + { + flush(); + fd = s.fd; + s.fd = -1; + warn = s.warn; + written = s.written; + return *this; + } + + ~FdSink(); + + void write(const unsigned char * data, size_t len) override; + + bool good() override; + +private: + bool _good = true; +}; + + +/* A source that reads data from a file descriptor. */ +struct FdSource : BufferedSource +{ + int fd; + size_t read = 0; + + FdSource() : fd(-1) { } + FdSource(int fd) : fd(fd) { } + FdSource(FdSource&&) = default; + + FdSource& operator=(FdSource && s) + { + fd = s.fd; + s.fd = -1; + read = s.read; + return *this; + } + + bool good() override; +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override; +private: + bool _good = true; +}; + + +/* A sink that writes data to a string. */ +struct StringSink : Sink +{ + ref s; + StringSink() : s(make_ref()) { }; + StringSink(ref s) : s(s) { }; + void operator () (const unsigned char * data, size_t len) override; +}; + + +/* A source that reads data from a string. */ +struct StringSource : Source +{ + const string & s; + size_t pos; + StringSource(const string & _s) : s(_s), pos(0) { } + size_t read(unsigned char * data, size_t len) override; +}; + + +/* Adapter class of a Source that saves all data read to `s'. */ +struct TeeSource : Source +{ + Source & orig; + ref data; + TeeSource(Source & orig) + : orig(orig), data(make_ref()) { } + size_t read(unsigned char * data, size_t len) + { + size_t n = orig.read(data, len); + this->data->append((const char *) data, n); + return n; + } +}; + +/* A reader that consumes the original Source until 'size'. */ +struct SizedSource : Source +{ + Source & orig; + size_t remain; + SizedSource(Source & orig, size_t size) + : orig(orig), remain(size) { } + size_t read(unsigned char * data, size_t len) + { + if (this->remain <= 0) { + throw EndOfFile("sized: unexpected end-of-file"); + } + len = std::min(len, this->remain); + size_t n = this->orig.read(data, len); + this->remain -= n; + return n; + } + + /* Consume the original source until no remain data is left to consume. */ + size_t drainAll() + { + std::vector buf(8192); + size_t sum = 0; + while (this->remain > 0) { + size_t n = read(buf.data(), buf.size()); + sum += n; + } + return sum; + } +}; + +/* Convert a function into a sink. */ +struct LambdaSink : Sink +{ + typedef std::function lambda_t; + + lambda_t lambda; + + LambdaSink(const lambda_t & lambda) : lambda(lambda) { } + + virtual void operator () (const unsigned char * data, size_t len) + { + lambda(data, len); + } +}; + + +/* Convert a function into a source. */ +struct LambdaSource : Source +{ + typedef std::function lambda_t; + + lambda_t lambda; + + LambdaSource(const lambda_t & lambda) : lambda(lambda) { } + + size_t read(unsigned char * data, size_t len) override + { + return lambda(data, len); + } +}; + + +/* Convert a function that feeds data into a Sink into a Source. The + Source executes the function as a coroutine. */ +std::unique_ptr sinkToSource( + std::function fun, + std::function eof = []() { + throw EndOfFile("coroutine has finished"); + }); + + +void writePadding(size_t len, Sink & sink); +void writeString(const unsigned char * buf, size_t len, Sink & sink); + +inline Sink & operator << (Sink & sink, uint64_t n) +{ + unsigned char buf[8]; + buf[0] = n & 0xff; + buf[1] = (n >> 8) & 0xff; + buf[2] = (n >> 16) & 0xff; + buf[3] = (n >> 24) & 0xff; + buf[4] = (n >> 32) & 0xff; + buf[5] = (n >> 40) & 0xff; + buf[6] = (n >> 48) & 0xff; + buf[7] = (unsigned char) (n >> 56) & 0xff; + sink(buf, sizeof(buf)); + return sink; +} + +Sink & operator << (Sink & sink, const string & s); +Sink & operator << (Sink & sink, const Strings & s); +Sink & operator << (Sink & sink, const StringSet & s); + + +MakeError(SerialisationError, Error) + + +template +T readNum(Source & source) +{ + unsigned char buf[8]; + source(buf, sizeof(buf)); + + uint64_t n = + ((unsigned long long) buf[0]) | + ((unsigned long long) buf[1] << 8) | + ((unsigned long long) buf[2] << 16) | + ((unsigned long long) buf[3] << 24) | + ((unsigned long long) buf[4] << 32) | + ((unsigned long long) buf[5] << 40) | + ((unsigned long long) buf[6] << 48) | + ((unsigned long long) buf[7] << 56); + + if (n > std::numeric_limits::max()) + throw SerialisationError("serialised integer %d is too large for type '%s'", n, typeid(T).name()); + + return (T) n; +} + + +inline unsigned int readInt(Source & source) +{ + return readNum(source); +} + + +inline uint64_t readLongLong(Source & source) +{ + return readNum(source); +} + + +void readPadding(size_t len, Source & source); +size_t readString(unsigned char * buf, size_t max, Source & source); +string readString(Source & source, size_t max = std::numeric_limits::max()); +template T readStrings(Source & source); + +Source & operator >> (Source & in, string & s); + +template +Source & operator >> (Source & in, T & n) +{ + n = readNum(in); + return in; +} + +template +Source & operator >> (Source & in, bool & b) +{ + b = readNum(in); + return in; +} + + +} diff --git a/third_party/nix/src/libutil/sync.hh b/third_party/nix/src/libutil/sync.hh new file mode 100644 index 000000000..e1d591d77 --- /dev/null +++ b/third_party/nix/src/libutil/sync.hh @@ -0,0 +1,88 @@ +#pragma once + +#include +#include +#include +#include + +namespace nix { + +/* This template class ensures synchronized access to a value of type + T. It is used as follows: + + struct Data { int x; ... }; + + Sync data; + + { + auto data_(data.lock()); + data_->x = 123; + } + + Here, "data" is automatically unlocked when "data_" goes out of + scope. +*/ + +template +class Sync +{ +private: + M mutex; + T data; + +public: + + Sync() { } + Sync(const T & data) : data(data) { } + Sync(T && data) noexcept : data(std::move(data)) { } + + class Lock + { + private: + Sync * s; + std::unique_lock lk; + friend Sync; + Lock(Sync * s) : s(s), lk(s->mutex) { } + public: + Lock(Lock && l) : s(l.s) { abort(); } + Lock(const Lock & l) = delete; + ~Lock() { } + T * operator -> () { return &s->data; } + T & operator * () { return s->data; } + + void wait(std::condition_variable & cv) + { + assert(s); + cv.wait(lk); + } + + template + std::cv_status wait_for(std::condition_variable & cv, + const std::chrono::duration & duration) + { + assert(s); + return cv.wait_for(lk, duration); + } + + template + bool wait_for(std::condition_variable & cv, + const std::chrono::duration & duration, + Predicate pred) + { + assert(s); + return cv.wait_for(lk, duration, pred); + } + + template + std::cv_status wait_until(std::condition_variable & cv, + const std::chrono::time_point & duration) + { + assert(s); + return cv.wait_until(lk, duration); + } + }; + + Lock lock() { return Lock(this); } +}; + +} diff --git a/third_party/nix/src/libutil/thread-pool.cc b/third_party/nix/src/libutil/thread-pool.cc new file mode 100644 index 000000000..857ee91f8 --- /dev/null +++ b/third_party/nix/src/libutil/thread-pool.cc @@ -0,0 +1,156 @@ +#include "thread-pool.hh" +#include "affinity.hh" + +namespace nix { + +ThreadPool::ThreadPool(size_t _maxThreads) + : maxThreads(_maxThreads) +{ + restoreAffinity(); // FIXME + + if (!maxThreads) { + maxThreads = std::thread::hardware_concurrency(); + if (!maxThreads) maxThreads = 1; + } + + debug("starting pool of %d threads", maxThreads - 1); +} + +ThreadPool::~ThreadPool() +{ + shutdown(); +} + +void ThreadPool::shutdown() +{ + std::vector workers; + { + auto state(state_.lock()); + quit = true; + std::swap(workers, state->workers); + } + + if (workers.empty()) return; + + debug("reaping %d worker threads", workers.size()); + + work.notify_all(); + + for (auto & thr : workers) + thr.join(); +} + +void ThreadPool::enqueue(const work_t & t) +{ + auto state(state_.lock()); + if (quit) + throw ThreadPoolShutDown("cannot enqueue a work item while the thread pool is shutting down"); + state->pending.push(t); + /* Note: process() also executes items, so count it as a worker. */ + if (state->pending.size() > state->workers.size() + 1 && state->workers.size() + 1 < maxThreads) + state->workers.emplace_back(&ThreadPool::doWork, this, false); + work.notify_one(); +} + +void ThreadPool::process() +{ + state_.lock()->draining = true; + + /* Do work until no more work is pending or active. */ + try { + doWork(true); + + auto state(state_.lock()); + + assert(quit); + + if (state->exception) + std::rethrow_exception(state->exception); + + } catch (...) { + /* In the exceptional case, some workers may still be + active. They may be referencing the stack frame of the + caller. So wait for them to finish. (~ThreadPool also does + this, but it might be destroyed after objects referenced by + the work item lambdas.) */ + shutdown(); + throw; + } +} + +void ThreadPool::doWork(bool mainThread) +{ + if (!mainThread) + interruptCheck = [&]() { return (bool) quit; }; + + bool didWork = false; + std::exception_ptr exc; + + while (true) { + work_t w; + { + auto state(state_.lock()); + + if (didWork) { + assert(state->active); + state->active--; + + if (exc) { + + if (!state->exception) { + state->exception = exc; + // Tell the other workers to quit. + quit = true; + work.notify_all(); + } else { + /* Print the exception, since we can't + propagate it. */ + try { + std::rethrow_exception(exc); + } catch (std::exception & e) { + if (!dynamic_cast(&e) && + !dynamic_cast(&e)) + ignoreException(); + } catch (...) { + } + } + } + } + + /* Wait until a work item is available or we're asked to + quit. */ + while (true) { + if (quit) return; + + if (!state->pending.empty()) break; + + /* If there are no active or pending items, and the + main thread is running process(), then no new items + can be added. So exit. */ + if (!state->active && state->draining) { + quit = true; + work.notify_all(); + return; + } + + state.wait(work); + } + + w = std::move(state->pending.front()); + state->pending.pop(); + state->active++; + } + + try { + w(); + } catch (...) { + exc = std::current_exception(); + } + + didWork = true; + } +} + +} + + diff --git a/third_party/nix/src/libutil/thread-pool.hh b/third_party/nix/src/libutil/thread-pool.hh new file mode 100644 index 000000000..bb16b639a --- /dev/null +++ b/third_party/nix/src/libutil/thread-pool.hh @@ -0,0 +1,143 @@ +#pragma once + +#include "sync.hh" +#include "util.hh" + +#include +#include +#include +#include +#include + +namespace nix { + +MakeError(ThreadPoolShutDown, Error) + +/* A simple thread pool that executes a queue of work items + (lambdas). */ +class ThreadPool +{ +public: + + ThreadPool(size_t maxThreads = 0); + + ~ThreadPool(); + + // FIXME: use std::packaged_task? + typedef std::function work_t; + + /* Enqueue a function to be executed by the thread pool. */ + void enqueue(const work_t & t); + + /* Execute work items until the queue is empty. Note that work + items are allowed to add new items to the queue; this is + handled correctly. Queue processing stops prematurely if any + work item throws an exception. This exception is propagated to + the calling thread. If multiple work items throw an exception + concurrently, only one item is propagated; the others are + printed on stderr and otherwise ignored. */ + void process(); + +private: + + size_t maxThreads; + + struct State + { + std::queue pending; + size_t active = 0; + std::exception_ptr exception; + std::vector workers; + bool draining = false; + }; + + std::atomic_bool quit{false}; + + Sync state_; + + std::condition_variable work; + + void doWork(bool mainThread); + + void shutdown(); +}; + +/* Process in parallel a set of items of type T that have a partial + ordering between them. Thus, any item is only processed after all + its dependencies have been processed. */ +template +void processGraph( + ThreadPool & pool, + const std::set & nodes, + std::function(const T &)> getEdges, + std::function processNode) +{ + struct Graph { + std::set left; + std::map> refs, rrefs; + }; + + Sync graph_(Graph{nodes, {}, {}}); + + std::function worker; + + worker = [&](const T & node) { + + { + auto graph(graph_.lock()); + auto i = graph->refs.find(node); + if (i == graph->refs.end()) + goto getRefs; + goto doWork; + } + + getRefs: + { + auto refs = getEdges(node); + refs.erase(node); + + { + auto graph(graph_.lock()); + for (auto & ref : refs) + if (graph->left.count(ref)) { + graph->refs[node].insert(ref); + graph->rrefs[ref].insert(node); + } + if (graph->refs[node].empty()) + goto doWork; + } + } + + return; + + doWork: + processNode(node); + + /* Enqueue work for all nodes that were waiting on this one + and have no unprocessed dependencies. */ + { + auto graph(graph_.lock()); + for (auto & rref : graph->rrefs[node]) { + auto & refs(graph->refs[rref]); + auto i = refs.find(node); + assert(i != refs.end()); + refs.erase(i); + if (refs.empty()) + pool.enqueue(std::bind(worker, rref)); + } + graph->left.erase(node); + graph->refs.erase(node); + graph->rrefs.erase(node); + } + }; + + for (auto & node : nodes) + pool.enqueue(std::bind(worker, std::ref(node))); + + pool.process(); + + if (!graph_.lock()->left.empty()) + throw Error("graph processing incomplete (cyclic reference?)"); +} + +} diff --git a/third_party/nix/src/libutil/types.hh b/third_party/nix/src/libutil/types.hh new file mode 100644 index 000000000..92bf469b5 --- /dev/null +++ b/third_party/nix/src/libutil/types.hh @@ -0,0 +1,150 @@ +#pragma once + + +#include "ref.hh" + +#include +#include +#include +#include +#include + +#include + +/* Before 4.7, gcc's std::exception uses empty throw() specifiers for + * its (virtual) destructor and what() in c++11 mode, in violation of spec + */ +#ifdef __GNUC__ +#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 7) +#define EXCEPTION_NEEDS_THROW_SPEC +#endif +#endif + + +namespace nix { + + +/* Inherit some names from other namespaces for convenience. */ +using std::string; +using std::list; +using std::set; +using std::vector; +using boost::format; + + +/* A variadic template that does nothing. Useful to call a function + for all variadic arguments but ignoring the result. */ +struct nop { template nop(T...) {} }; + + +struct FormatOrString +{ + string s; + FormatOrString(const string & s) : s(s) { }; + FormatOrString(const format & f) : s(f.str()) { }; + FormatOrString(const char * s) : s(s) { }; +}; + + +/* A helper for formatting strings. ‘fmt(format, a_0, ..., a_n)’ is + equivalent to ‘boost::format(format) % a_0 % ... % + ... a_n’. However, ‘fmt(s)’ is equivalent to ‘s’ (so no %-expansion + takes place). */ + +inline std::string fmt(const std::string & s) +{ + return s; +} + +inline std::string fmt(const char * s) +{ + return s; +} + +inline std::string fmt(const FormatOrString & fs) +{ + return fs.s; +} + +template +inline std::string fmt(const std::string & fs, Args... args) +{ + boost::format f(fs); + f.exceptions(boost::io::all_error_bits ^ boost::io::too_many_args_bit); + nop{boost::io::detail::feed(f, args)...}; + return f.str(); +} + + +/* BaseError should generally not be caught, as it has Interrupted as + a subclass. Catch Error instead. */ +class BaseError : public std::exception +{ +protected: + string prefix_; // used for location traces etc. + string err; +public: + unsigned int status = 1; // exit status + + template + BaseError(unsigned int status, Args... args) + : err(fmt(args...)) + , status(status) + { + } + + template + BaseError(Args... args) + : err(fmt(args...)) + { + } + +#ifdef EXCEPTION_NEEDS_THROW_SPEC + ~BaseError() throw () { }; + const char * what() const throw () { return err.c_str(); } +#else + const char * what() const noexcept { return err.c_str(); } +#endif + + const string & msg() const { return err; } + const string & prefix() const { return prefix_; } + BaseError & addPrefix(const FormatOrString & fs); +}; + +#define MakeError(newClass, superClass) \ + class newClass : public superClass \ + { \ + public: \ + using superClass::superClass; \ + }; + +MakeError(Error, BaseError) + +class SysError : public Error +{ +public: + int errNo; + + template + SysError(Args... args) + : Error(addErrno(fmt(args...))) + { } + +private: + + std::string addErrno(const std::string & s); +}; + + +typedef list Strings; +typedef set StringSet; +typedef std::map StringMap; + + +/* Paths are just strings. */ +typedef string Path; +typedef list Paths; +typedef set PathSet; + + +} diff --git a/third_party/nix/src/libutil/util.cc b/third_party/nix/src/libutil/util.cc new file mode 100644 index 000000000..ad8cc1894 --- /dev/null +++ b/third_party/nix/src/libutil/util.cc @@ -0,0 +1,1552 @@ +#include "lazy.hh" +#include "util.hh" +#include "affinity.hh" +#include "sync.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __APPLE__ +#include +#endif + +#ifdef __linux__ +#include +#endif + + +extern char * * environ; + + +namespace nix { + + +const std::string nativeSystem = SYSTEM; + + +BaseError & BaseError::addPrefix(const FormatOrString & fs) +{ + prefix_ = fs.s + prefix_; + return *this; +} + + +std::string SysError::addErrno(const std::string & s) +{ + errNo = errno; + return s + ": " + strerror(errNo); +} + + +string getEnv(const string & key, const string & def) +{ + char * value = getenv(key.c_str()); + return value ? string(value) : def; +} + + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(std::map newEnv) +{ + clearEnv(); + for (auto newEnvVar : newEnv) + { + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + } +} + + +Path absPath(Path path, Path dir) +{ + if (path[0] != '/') { + if (dir == "") { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + dir = buf; +#ifdef __GNU__ + free(buf); +#endif + } + path = dir + "/" + path; + } + return canonPath(path); +} + + +Path canonPath(const Path & path, bool resolveSymlinks) +{ + assert(path != ""); + + string s; + + if (path[0] != '/') + throw Error(format("not an absolute path: '%1%'") % path); + + string::const_iterator i = path.begin(), end = path.end(); + string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (i != end && *i == '/') i++; + if (i == end) break; + + /* Ignore `.'. */ + if (*i == '.' && (i + 1 == end || i[1] == '/')) + i++; + + /* If `..', delete the last component. */ + else if (*i == '.' && i + 1 < end && i[1] == '.' && + (i + 2 == end || i[2] == '/')) + { + if (!s.empty()) s.erase(s.rfind('/')); + i += 2; + } + + /* Normal component; copy it. */ + else { + s += '/'; + while (i != end && *i != '/') s += *i++; + + /* If s points to a symlink, resolve it and restart (since + the symlink target might contain new symlinks). */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error(format("infinite symlink recursion in path '%1%'") % path); + temp = absPath(readLink(s), dirOf(s)) + + string(i, end); + i = temp.begin(); /* restart */ + end = temp.end(); + s = ""; + } + } + } + + return s.empty() ? "/" : s; +} + + +Path dirOf(const Path & path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +string baseNameOf(const Path & path) +{ + if (path.empty()) + return ""; + + Path::size_type last = path.length() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + Path::size_type pos = path.rfind('/', last); + if (pos == string::npos) + pos = 0; + else + pos += 1; + + return string(path, pos, last - pos + 1); +} + + +bool isInDir(const Path & path, const Path & dir) +{ + return path[0] == '/' + && string(path, 0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(const Path & path, const Path & dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of '%1%'") % path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError(format("getting status of %1%") % path); + return false; +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError(format("opening directory '%1%'") % path); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir.get())) { /* sic */ + checkInterrupt(); + string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError(format("reading directory '%1%'") % path); + + return entries; +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + std::vector buf(st.st_size); + readFull(fd, buf.data(), st.st_size); + + return string((char *) buf.data(), st.st_size); +} + + +string readFile(const Path & path, bool drain) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + return drain ? drainFD(fd.get()) : readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, const string & s, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + writeFull(fd.get(), s); +} + + +void writeFile(const Path & path, Source & source, mode_t mode) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError(format("opening file '%1%'") % path); + + std::vector buf(64 * 1024); + + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), (unsigned char *) buf.data(), n); + } catch (EndOfFile &) { break; } + } +} + + +string readLine(int fd) +{ + string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +static void _deletePath(const Path & path, unsigned long long & bytesFreed) +{ + checkInterrupt(); + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + if (errno == ENOENT) return; + throw SysError(format("getting status of '%1%'") % path); + } + + if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) + bytesFreed += st.st_size; + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (chmod(path.c_str(), st.st_mode | PERM_MASK) == -1) + throw SysError(format("chmod '%1%'") % path); + } + + for (auto & i : readDirectory(path)) + _deletePath(path + "/" + i.name, bytesFreed); + } + + if (remove(path.c_str()) == -1) { + if (errno == ENOENT) return; + throw SysError(format("cannot unlink '%1%'") % path); + } +} + + +void deletePath(const Path & path) +{ + unsigned long long dummy; + deletePath(path, dummy); +} + + +void deletePath(const Path & path, unsigned long long & bytesFreed) +{ + //Activity act(*logger, lvlDebug, format("recursively deleting path '%1%'") % path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + int & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR", "/tmp") : tmpRoot, true); + if (includePid) + return (format("%1%/%2%-%3%-%4%") % tmpRoot % prefix % getpid() % counter++).str(); + else + return (format("%1%/%2%-%3%") % tmpRoot % prefix % counter++).str(); +} + + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static int globalCounter = 0; + int localCounter = 0; + int & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError(format("setting group of directory '%1%'") % tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError(format("creating directory '%1%'") % tmpDir); + } +} + + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER", ""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + + +static Lazy getHome2([]() { + Path homeDir = getEnv("HOME"); + if (homeDir.empty()) { + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(geteuid(), &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + homeDir = pw->pw_dir; + } + return homeDir; +}); + +Path getHome() { return getHome2(); } + + +Path getCacheDir() +{ + Path cacheDir = getEnv("XDG_CACHE_HOME"); + if (cacheDir.empty()) + cacheDir = getHome() + "/.cache"; + return cacheDir; +} + + +Path getConfigDir() +{ + Path configDir = getEnv("XDG_CONFIG_HOME"); + if (configDir.empty()) + configDir = getHome() + "/.config"; + return configDir; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + string configDirs = getEnv("XDG_CONFIG_DIRS"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + Path dataDir = getEnv("XDG_DATA_HOME"); + if (dataDir.empty()) + dataDir = getHome() + "/.local/share"; + return dataDir; +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError(format("creating directory '%1%'") % path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError(format("statting symlink '%1%'") % path); + + if (!S_ISDIR(st.st_mode)) throw Error(format("'%1%' is not a directory") % path); + + return created; +} + + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); +} + + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + if (rename(tmp.c_str(), link.c_str()) != 0) + throw SysError(format("renaming '%1%' to '%2%'") % tmp % link); + + break; + } +} + + +void readFull(int fd, unsigned char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, (char *) buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts) +{ + while (count) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, (char *) buf, count); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) { + count -= res; + buf += res; + } + } +} + + +void writeFull(int fd, const string & s, bool allowInterrupts) +{ + writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts); +} + + +string drainFD(int fd, bool block) +{ + StringSink sink; + drainFD(fd, sink, block); + return std::move(*sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + int saved; + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink(buf.data(), rd); + } +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError(format("cannot unlink '%1%'") % path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + + + +////////////////////////////////////////////////////////////////////// + + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError(format("closing file descriptor %1%") % fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + + +////////////////////////////////////////////////////////////////////// + + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug(format("killing process %1%") % pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + printError((SysError("killing process %d", pid).msg())); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get child exit status"); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug(format("killing all processes running under uid '%1%'") % uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + ProcessOptions options; + options.allowVfork = false; + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH) break; /* no more processes */ + if (errno != EINTR) + throw SysError(format("cannot kill processes for uid '%1%'") % uid); + } + + _exit(0); + }, options); + + int status = pid.wait(); + if (status != 0) + throw Error(format("cannot kill processes for uid '%1%': %2%") % uid % statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + auto wrapper = [&]() { + if (!options.allowVfork) + logger = makeDefaultLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + restoreAffinity(); + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = doFork(options.allowVfork, wrapper); + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::vector stringsToCharPtrs(const Strings & ss) +{ + std::vector res; + for (auto & s : ss) res.push_back((char *) s.c_str()); + res.push_back(0); + return res; +} + + +string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input) +{ + RunOptions opts(program, args); + opts.searchPath = searchPath; + opts.input = input; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + +std::pair runProgram(const RunOptions & options_) +{ + RunOptions options(options_); + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(*sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + if (options.environment) + processOptions.allowVfork = false; + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreSignals(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide = -1; + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide = -1; + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), buf.data(), n); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide = -1; + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, fmt("program '%1%' %2%", options.program, statusToString(status))); +} + + +void closeMostFDs(const set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + + +////////////////////////////////////////////////////////////////////// + + +bool _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exception()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +template C tokenizeString(const string & s, const string & separators) +{ + C result; + string::size_type pos = s.find_first_not_of(separators, 0); + while (pos != string::npos) { + string::size_type end = s.find_first_of(separators, pos + 1); + if (end == string::npos) end = s.size(); + string token(s, pos, end - pos); + result.insert(result.end(), token); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template Strings tokenizeString(const string & s, const string & separators); +template StringSet tokenizeString(const string & s, const string & separators); +template vector tokenizeString(const string & s, const string & separators); + + +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +string concatStringsSep(const string & sep, const StringSet & ss) +{ + string s; + for (auto & i : ss) { + if (s.size() != 0) s += sep; + s += i; + } + return s; +} + + +string chomp(const string & s) +{ + size_t i = s.find_last_not_of(" \n\r\t"); + return i == string::npos ? "" : string(s, 0, i + 1); +} + + +string trim(const string & s, const string & whitespace) +{ + auto i = s.find_first_not_of(whitespace); + if (i == string::npos) return ""; + auto j = s.find_last_not_of(whitespace); + return string(s, i, j == string::npos ? j : j - i + 1); +} + + +string replaceStrings(const std::string & s, + const std::string & from, const std::string & to) +{ + if (from.empty()) return s; + string res = s; + size_t pos = 0; + while ((pos = res.find(from, pos)) != std::string::npos) { + res.replace(pos, from.size(), to); + pos += to.size(); + } + return res; +} + + +string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return (format("failed with exit code %1%") % WEXITSTATUS(status)).str(); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return (format("failed due to signal %1% (%2%)") % sig % description).str(); +#else + return (format("failed due to signal %1%") % sig).str(); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + + +bool hasPrefix(const string & s, const string & prefix) +{ + return s.compare(0, prefix.size(), prefix) == 0; +} + + +bool hasSuffix(const string & s, const string & suffix) +{ + return s.size() >= suffix.size() && string(s, s.size() - suffix.size()) == suffix; +} + + +std::string toLower(const std::string & s) +{ + std::string r(s); + for (auto & c : r) + c = std::tolower(c); + return r; +} + + +std::string shellEscape(const std::string & s) +{ + std::string r = "'"; + for (auto & i : s) + if (i == '\'') r += "'\\''"; else r += i; + r += '\''; + return r; +} + + +void ignoreException() +{ + try { + throw; + } catch (std::exception & e) { + printError(format("error (ignored): %1%") % e.what()); + } +} + + +std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r') + // do nothing for now + i++; + + else { + t += *i++; w++; + } + } + + return t; +} + + +static char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + +string base64Encode(const string & s) +{ + string res; + int data = 0, nbits = 0; + + for (char c : s) { + data = data << 8 | (unsigned char) c; + nbits += 8; + while (nbits >= 6) { + nbits -= 6; + res.push_back(base64Chars[data >> nbits & 0x3f]); + } + } + + if (nbits) res.push_back(base64Chars[data << (6 - nbits) & 0x3f]); + while (res.size() % 4) res.push_back('='); + + return res; +} + + +string base64Decode(const string & s) +{ + bool init = false; + char decode[256]; + if (!init) { + // FIXME: not thread-safe. + memset(decode, -1, sizeof(decode)); + for (int i = 0; i < 64; i++) + decode[(int) base64Chars[i]] = i; + init = true; + } + + string res; + unsigned int d = 0, bits = 0; + + for (char c : s) { + if (c == '=') break; + if (c == '\n') continue; + + char digit = decode[(unsigned char) c]; + if (digit == -1) + throw Error("invalid character in Base64 string"); + + bits += 6; + d = d << 6 | digit; + if (bits >= 8) { + res.push_back(d >> (bits - 8) & 0xff); + bits -= 8; + } + } + + return res; +} + + +void callFailure(const std::function & failure, std::exception_ptr exc) +{ + try { + failure(exc); + } catch (std::exception & e) { + printError(format("uncaught exception: %s") % e.what()); + abort(); + } +} + + +static Sync> windowSize{{0, 0}}; + + +static void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + + +static Sync>> _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + auto interruptCallbacks(_interruptCallbacks.lock()); + for (auto & callback : *interruptCallbacks) { + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + +static sigset_t savedSignalMask; + +void startSignalHandlerThread() +{ + updateWindowSize(); + + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("quering signal mask"); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + std::list>::iterator it; + ~InterruptCallbackImpl() override + { + _interruptCallbacks.lock()->erase(it); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->push_back(callback); + + auto res = std::make_unique(); + res->it = interruptCallbacks->end(); + res->it--; + + return std::unique_ptr(res.release()); +} + +} diff --git a/third_party/nix/src/libutil/util.hh b/third_party/nix/src/libutil/util.hh new file mode 100644 index 000000000..f057fdb2c --- /dev/null +++ b/third_party/nix/src/libutil/util.hh @@ -0,0 +1,542 @@ +#pragma once + +#include "types.hh" +#include "logging.hh" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + + +/* The system for which Nix is compiled. */ +extern const std::string nativeSystem; + + +/* Return an environment variable. */ +string getEnv(const string & key, const string & def = ""); + +/* Get the entire environment. */ +std::map getEnv(); + +/* Clear the environment. */ +void clearEnv(); + +/* Return an absolutized path, resolving paths relative to the + specified directory, or the current directory otherwise. The path + is also canonicalised. */ +Path absPath(Path path, Path dir = ""); + +/* Canonicalise a path by removing all `.' or `..' components and + double or trailing slashes. Optionally resolves all symlink + components such that each component of the resulting path is *not* + a symbolic link. */ +Path canonPath(const Path & path, bool resolveSymlinks = false); + +/* Return the directory part of the given canonical path, i.e., + everything before the final `/'. If the path is the root or an + immediate child thereof (e.g., `/foo'), this means an empty string + is returned. */ +Path dirOf(const Path & path); + +/* Return the base name of the given canonical path, i.e., everything + following the final `/'. */ +string baseNameOf(const Path & path); + +/* Check whether 'path' is a descendant of 'dir'. */ +bool isInDir(const Path & path, const Path & dir); + +/* Check whether 'path' is equal to 'dir' or a descendant of 'dir'. */ +bool isDirOrInDir(const Path & path, const Path & dir); + +/* Get status of `path'. */ +struct stat lstat(const Path & path); + +/* Return true iff the given path exists. */ +bool pathExists(const Path & path); + +/* Read the contents (target) of a symbolic link. The result is not + in any way canonicalised. */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/* Read the contents of a directory. The entries `.' and `..' are + removed. */ +struct DirEntry +{ + string name; + ino_t ino; + unsigned char type; // one of DT_* + DirEntry(const string & name, ino_t ino, unsigned char type) + : name(name), ino(ino), type(type) { } +}; + +typedef vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/* Read the contents of a file into a string. */ +string readFile(int fd); +string readFile(const Path & path, bool drain = false); +void readFile(const Path & path, Sink & sink); + +/* Write a string to a file. */ +void writeFile(const Path & path, const string & s, mode_t mode = 0666); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666); + +/* Read a line from a file descriptor. */ +string readLine(int fd); + +/* Write a line to a file descriptor. */ +void writeLine(int fd, string s); + +/* Delete a path; i.e., in the case of a directory, it is deleted + recursively. It's not an error if the path does not exist. The + second variant returns the number of bytes and blocks freed. */ +void deletePath(const Path & path); + +void deletePath(const Path & path, unsigned long long & bytesFreed); + +/* Create a temporary directory. */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +std::string getUserName(); + +/* Return $HOME or the user's home directory from /etc/passwd. */ +Path getHome(); + +/* Return $XDG_CACHE_HOME or $HOME/.cache. */ +Path getCacheDir(); + +/* Return $XDG_CONFIG_HOME or $HOME/.config. */ +Path getConfigDir(); + +/* Return the directories to search for user configuration files */ +std::vector getConfigDirs(); + +/* Return $XDG_DATA_HOME or $HOME/.local/share. */ +Path getDataDir(); + +/* Create a directory and all its parents, if necessary. Returns the + list of created directories, in order of creation. */ +Paths createDirs(const Path & path); + +/* Create a symlink. */ +void createSymlink(const Path & target, const Path & link); + +/* Atomically create or replace a symlink. */ +void replaceSymlink(const Path & target, const Path & link); + + +/* Wrappers arount read()/write() that read/write exactly the + requested number of bytes. */ +void readFull(int fd, unsigned char * buf, size_t count); +void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true); +void writeFull(int fd, const string & s, bool allowInterrupts = true); + +MakeError(EndOfFile, Error) + + +/* Read a file descriptor until EOF occurs. */ +string drainFD(int fd, bool block = true); + +void drainFD(int fd, Sink & sink, bool block = true); + + +/* Automatic cleanup of resources. */ + + +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } +}; + + +class AutoCloseFD +{ + int fd; + void close(); +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); +}; + + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/* Kill all processes running under the specified uid by sending them + a SIGKILL. */ +void killUser(uid_t uid); + + +/* Fork a process that runs the given function, and return the child + pid to the caller. */ +struct ProcessOptions +{ + string errorPrefix = "error: "; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = true; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/* Run a program and return its stdout in a string (i.e., like the + shell backtick operator). */ +string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}); + +struct RunOptions +{ + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + Path program; + bool searchPath = true; + Strings args; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool _killStderr = false; + + RunOptions(const Path & program, const Strings & args) + : program(program), args(args) { }; + + RunOptions & killStderr(bool v) { _killStderr = true; return *this; } +}; + +std::pair runProgram(const RunOptions & options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, Args... args) + : Error(args...), status(status) + { } +}; + +/* Convert a list of strings to a null-terminated vector of char + *'s. The result must not be accessed beyond the lifetime of the + list of strings. */ +std::vector stringsToCharPtrs(const Strings & ss); + +/* Close all file descriptors except those listed in the given set. + Good practice in child processes. */ +void closeMostFDs(const set & exceptions); + +/* Set the close-on-exec flag for the given file descriptor. */ +void closeOnExec(int fd); + + +/* User interruption. */ + +extern bool _isInterrupted; + +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError) + + +MakeError(FormatError, Error) + + +/* String tokenizer. */ +template C tokenizeString(const string & s, const string & separators = " \t\n\r"); + + +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); +string concatStringsSep(const string & sep, const StringSet & ss); + + +/* Remove trailing whitespace from a string. */ +string chomp(const string & s); + + +/* Remove whitespace from the start and end of a string. */ +string trim(const string & s, const string & whitespace = " \n\r\t"); + + +/* Replace all occurrences of a string inside another string. */ +string replaceStrings(const std::string & s, + const std::string & from, const std::string & to); + + +/* Convert the exit status of a child as returned by wait() into an + error string. */ +string statusToString(int status); + +bool statusOk(int status); + + +/* Parse a string into an integer. */ +template bool string2Int(const string & s, N & n) +{ + if (string(s, 0, 1) == "-" && !std::numeric_limits::is_signed) + return false; + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + +/* Parse a string into a float. */ +template bool string2Float(const string & s, N & n) +{ + std::istringstream str(s); + str >> n; + return str && str.get() == EOF; +} + + +/* Return true iff `s' starts with `prefix'. */ +bool hasPrefix(const string & s, const string & prefix); + + +/* Return true iff `s' ends in `suffix'. */ +bool hasSuffix(const string & s, const string & suffix); + + +/* Convert a string to lower case. */ +std::string toLower(const std::string & s); + + +/* Escape a string as a shell word. */ +std::string shellEscape(const std::string & s); + + +/* Exception handling in destructors: print an error message, then + ignore the exception. */ +void ignoreException(); + + +/* Some ANSI escape sequences. */ +#define ANSI_NORMAL "\e[0m" +#define ANSI_BOLD "\e[1m" +#define ANSI_FAINT "\e[2m" +#define ANSI_RED "\e[31;1m" +#define ANSI_GREEN "\e[32;1m" +#define ANSI_BLUE "\e[34;1m" + + +/* Truncate a string to 'width' printable characters. If 'filterAll' + is true, all ANSI escape sequences are filtered out. Otherwise, + some escape sequences (such as colour setting) are copied but not + included in the character count. Also, tabs are expanded to + spaces. */ +std::string filterANSIEscapes(const std::string & s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + + +/* Base64 encoding/decoding. */ +string base64Encode(const string & s); +string base64Decode(const string & s); + + +/* Get a value for the specified key from an associate container, or a + default value if the key doesn't exist. */ +template +string get(const T & map, const string & key, const string & def = "") +{ + auto i = map.find(key); + return i == map.end() ? def : i->second; +} + + +/* A callback is a wrapper around a lambda that accepts a valid of + type T or an exception. (We abuse std::future to pass the value or + exception.) */ +template +class Callback +{ + std::function)> fun; + std::atomic_flag done = ATOMIC_FLAG_INIT; + +public: + + Callback(std::function)> fun) : fun(fun) { } + + Callback(Callback && callback) : fun(std::move(callback.fun)) + { + auto prev = callback.done.test_and_set(); + if (prev) done.test_and_set(); + } + + void operator()(T && t) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_value(std::move(t)); + fun(promise.get_future()); + } + + void rethrow(const std::exception_ptr & exc = std::current_exception()) noexcept + { + auto prev = done.test_and_set(); + assert(!prev); + std::promise promise; + promise.set_exception(exc); + fun(promise.get_future()); + } +}; + + +/* Start a thread that handles various signals. Also block those signals + on the current thread (and thus any threads created by it). */ +void startSignalHandlerThread(); + +/* Restore default signal handling. */ +void restoreSignals(); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/* Register a function that gets called on SIGINT (in a non-signal + context). */ +std::unique_ptr createInterruptCallback( + std::function callback); + +void triggerInterrupt(); + +/* A RAII class that causes the current thread to receive SIGUSR1 when + the signal handler thread receives SIGINT. That is, this allows + SIGINT to be multiplexed to multiple threads. */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + + +/* A RAII helper that increments a counter on construction and + decrements it on destruction. */ +template +struct MaintainCount +{ + T & counter; + long delta; + MaintainCount(T & counter, long delta = 1) : counter(counter), delta(delta) { counter += delta; } + ~MaintainCount() { counter -= delta; } +}; + + +/* Return the number of rows and columns of the terminal. */ +std::pair getWindowSize(); + + +/* Used in various places. */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + + +} diff --git a/third_party/nix/src/libutil/xml-writer.cc b/third_party/nix/src/libutil/xml-writer.cc new file mode 100644 index 000000000..e5cc2e9fc --- /dev/null +++ b/third_party/nix/src/libutil/xml-writer.cc @@ -0,0 +1,94 @@ +#include + +#include "xml-writer.hh" + + +namespace nix { + + +XMLWriter::XMLWriter(bool indent, std::ostream & output) + : output(output), indent(indent) +{ + output << "" << std::endl; + closed = false; +} + + +XMLWriter::~XMLWriter() +{ + close(); +} + + +void XMLWriter::close() +{ + if (closed) return; + while (!pendingElems.empty()) closeElement(); + closed = true; +} + + +void XMLWriter::indent_(size_t depth) +{ + if (!indent) return; + output << string(depth * 2, ' '); +} + + +void XMLWriter::openElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << ">"; + if (indent) output << std::endl; + pendingElems.push_back(name); +} + + +void XMLWriter::closeElement() +{ + assert(!pendingElems.empty()); + indent_(pendingElems.size() - 1); + output << ""; + if (indent) output << std::endl; + pendingElems.pop_back(); + if (pendingElems.empty()) closed = true; +} + + +void XMLWriter::writeEmptyElement(const string & name, + const XMLAttrs & attrs) +{ + assert(!closed); + indent_(pendingElems.size()); + output << "<" << name; + writeAttrs(attrs); + output << " />"; + if (indent) output << std::endl; +} + + +void XMLWriter::writeAttrs(const XMLAttrs & attrs) +{ + for (auto & i : attrs) { + output << " " << i.first << "=\""; + for (size_t j = 0; j < i.second.size(); ++j) { + char c = i.second[j]; + if (c == '"') output << """; + else if (c == '<') output << "<"; + else if (c == '>') output << ">"; + else if (c == '&') output << "&"; + /* Escape newlines to prevent attribute normalisation (see + XML spec, section 3.3.3. */ + else if (c == '\n') output << " "; + else output << c; + } + output << "\""; + } +} + + +} diff --git a/third_party/nix/src/libutil/xml-writer.hh b/third_party/nix/src/libutil/xml-writer.hh new file mode 100644 index 000000000..b98b44526 --- /dev/null +++ b/third_party/nix/src/libutil/xml-writer.hh @@ -0,0 +1,69 @@ +#pragma once + +#include +#include +#include +#include + + +namespace nix { + +using std::string; +using std::map; +using std::list; + + +typedef map XMLAttrs; + + +class XMLWriter +{ +private: + + std::ostream & output; + + bool indent; + bool closed; + + list pendingElems; + +public: + + XMLWriter(bool indent, std::ostream & output); + ~XMLWriter(); + + void close(); + + void openElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + void closeElement(); + + void writeEmptyElement(const string & name, + const XMLAttrs & attrs = XMLAttrs()); + +private: + void writeAttrs(const XMLAttrs & attrs); + + void indent_(size_t depth); +}; + + +class XMLOpenElement +{ +private: + XMLWriter & writer; +public: + XMLOpenElement(XMLWriter & writer, const string & name, + const XMLAttrs & attrs = XMLAttrs()) + : writer(writer) + { + writer.openElement(name, attrs); + } + ~XMLOpenElement() + { + writer.closeElement(); + } +}; + + +} diff --git a/third_party/nix/src/nix-build/nix-build.cc b/third_party/nix/src/nix-build/nix-build.cc new file mode 100755 index 000000000..33ad28704 --- /dev/null +++ b/third_party/nix/src/nix-build/nix-build.cc @@ -0,0 +1,508 @@ +#include +#include +#include +#include +#include +#include + +#include "store-api.hh" +#include "globals.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "util.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "get-drvs.hh" +#include "common-eval-args.hh" +#include "attr-path.hh" +#include "legacy.hh" + +using namespace nix; +using namespace std::string_literals; + +extern char * * environ; + +/* Recreate the effect of the perl shellwords function, breaking up a + * string into arguments like a shell word, including escapes + */ +std::vector shellwords(const string & s) +{ + std::regex whitespace("^(\\s+).*"); + auto begin = s.cbegin(); + std::vector res; + std::string cur; + enum state { + sBegin, + sQuote + }; + state st = sBegin; + auto it = begin; + for (; it != s.cend(); ++it) { + if (st == sBegin) { + std::smatch match; + if (regex_search(it, s.cend(), match, whitespace)) { + cur.append(begin, it); + res.push_back(cur); + cur.clear(); + it = match[1].second; + begin = it; + } + } + switch (*it) { + case '"': + cur.append(begin, it); + begin = it + 1; + st = st == sBegin ? sQuote : sBegin; + break; + case '\\': + /* perl shellwords mostly just treats the next char as part of the string with no special processing */ + cur.append(begin, it); + begin = ++it; + break; + } + } + cur.append(begin, it); + if (!cur.empty()) res.push_back(cur); + return res; +} + +static void _main(int argc, char * * argv) +{ + auto dryRun = false; + auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto pure = false; + auto fromArgs = false; + auto packages = false; + // Same condition as bash uses for interactive shells + auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); + Strings attrPaths; + Strings left; + RepairFlag repair = NoRepair; + Path gcRoot; + BuildMode buildMode = bmNormal; + bool readStdin = false; + + std::string envCommand; // interactive shell + Strings envExclude; + + auto myName = runEnv ? "nix-shell" : "nix-build"; + + auto inShebang = false; + std::string script; + std::vector savedArgs; + + AutoDelete tmpDir(createTempDir("", myName)); + + std::string outLink = "./result"; + + // List of environment variables kept for --pure + std::set keepVars{"HOME", "USER", "LOGNAME", "DISPLAY", "PATH", "TERM", "IN_NIX_SHELL", "TZ", "PAGER", "NIX_BUILD_SHELL", "SHLVL"}; + + Strings args; + for (int i = 1; i < argc; ++i) + args.push_back(argv[i]); + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + if (runEnv && argc > 1 && !std::regex_search(argv[1], std::regex("nix-shell"))) { + script = argv[1]; + try { + auto lines = tokenizeString(readFile(script), "\n"); + if (std::regex_search(lines.front(), std::regex("^#!"))) { + lines.pop_front(); + inShebang = true; + for (int i = 2; i < argc; ++i) + savedArgs.push_back(argv[i]); + args.clear(); + for (auto line : lines) { + line = chomp(line); + std::smatch match; + if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell (.*)$"))) + for (const auto & word : shellwords(match[1].str())) + args.push_back(word); + } + } + } catch (SysError &) { } + } + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") { + deletePath(tmpDir); + showManPage(myName); + } + + else if (*arg == "--version") + printVersion(myName); + + else if (*arg == "--add-drv-link" || *arg == "--indirect") + ; // obsolete + + else if (*arg == "--no-out-link" || *arg == "--no-link") + outLink = (Path) tmpDir + "/result"; + + else if (*arg == "--attr" || *arg == "-A") + attrPaths.push_back(getArg(*arg, arg, end)); + + else if (*arg == "--drv-link") + getArg(*arg, arg, end); // obsolete + + else if (*arg == "--out-link" || *arg == "-o") + outLink = getArg(*arg, arg, end); + + else if (*arg == "--add-root") + gcRoot = getArg(*arg, arg, end); + + else if (*arg == "--dry-run") + dryRun = true; + + else if (*arg == "--repair") { + repair = Repair; + buildMode = bmRepair; + } + + else if (*arg == "--run-env") // obsolete + runEnv = true; + + else if (*arg == "--command" || *arg == "--run") { + if (*arg == "--run") + interactive = false; + envCommand = getArg(*arg, arg, end) + "\nexit"; + } + + else if (*arg == "--check") + buildMode = bmCheck; + + else if (*arg == "--exclude") + envExclude.push_back(getArg(*arg, arg, end)); + + else if (*arg == "--expr" || *arg == "-E") + fromArgs = true; + + else if (*arg == "--pure") pure = true; + else if (*arg == "--impure") pure = false; + + else if (*arg == "--packages" || *arg == "-p") + packages = true; + + else if (inShebang && *arg == "-i") { + auto interpreter = getArg(*arg, arg, end); + interactive = false; + auto execArgs = ""; + + // Überhack to support Perl. Perl examines the shebang and + // executes it unless it contains the string "perl" or "indir", + // or (undocumented) argv[0] does not contain "perl". Exploit + // the latter by doing "exec -a". + if (std::regex_search(interpreter, std::regex("perl"))) + execArgs = "-a PERL"; + + std::ostringstream joined; + for (const auto & i : savedArgs) + joined << shellEscape(i) << ' '; + + if (std::regex_search(interpreter, std::regex("ruby"))) { + // Hack for Ruby. Ruby also examines the shebang. It tries to + // read the shebang to understand which packages to read from. Since + // this is handled via nix-shell -p, we wrap our ruby script execution + // in ruby -e 'load' which ignores the shebangs. + envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); + } else { + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); + } + } + + else if (*arg == "--keep") + keepVars.insert(getArg(*arg, arg, end)); + + else if (*arg == "-") + readStdin = true; + + else if (*arg != "" && arg->at(0) == '-') + return false; + + else + left.push_back(*arg); + + return true; + }); + + myArgs.parseCmdline(args); + + initPlugins(); + + if (packages && fromArgs) + throw UsageError("'-p' and '-E' are mutually exclusive"); + + auto store = openStore(); + + auto state = std::make_unique(myArgs.searchPath, store); + state->repair = repair; + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + if (packages) { + std::ostringstream joined; + joined << "with import { }; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; + for (const auto & i : left) + joined << '(' << i << ") "; + joined << "]; } \"\""; + fromArgs = true; + left = {joined.str()}; + } else if (!fromArgs) { + if (left.empty() && runEnv && pathExists("shell.nix")) + left = {"shell.nix"}; + if (left.empty()) + left = {"default.nix"}; + } + + if (runEnv) + setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); + + DrvInfos drvs; + + /* Parse the expressions. */ + std::vector exprs; + + if (readStdin) + exprs = {state->parseStdin()}; + else + for (auto i : left) { + if (fromArgs) + exprs.push_back(state->parseExprFromString(i, absPath("."))); + else { + auto absolute = i; + try { + absolute = canonPath(absPath(i), true); + } catch (Error & e) {}; + if (store->isStorePath(absolute) && std::regex_match(absolute, std::regex(".*\\.drv(!.*)?"))) + drvs.push_back(DrvInfo(*state, store, absolute)); + else + /* If we're in a #! script, interpret filenames + relative to the script. */ + exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + } + } + + /* Evaluate them into derivations. */ + if (attrPaths.empty()) attrPaths = {""}; + + for (auto e : exprs) { + Value vRoot; + state->eval(e, vRoot); + + for (auto & i : attrPaths) { + Value & v(*findAlongAttrPath(*state, i, autoArgs, vRoot)); + state->forceValue(v); + getDerivations(*state, v, "", autoArgs, drvs, false); + } + } + + state->printStats(); + + auto buildPaths = [&](const PathSet & paths) { + /* Note: we do this even when !printMissing to efficiently + fetch binary cache data. */ + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(paths, + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (settings.printMissing) + printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (!dryRun) + store->buildPaths(paths, buildMode); + }; + + if (runEnv) { + if (drvs.size() != 1) + throw UsageError("nix-shell requires a single derivation"); + + auto & drvInfo = drvs.front(); + auto drv = store->derivationFromPath(drvInfo.queryDrvPath()); + + PathSet pathsToBuild; + + /* Figure out what bash shell to use. If $NIX_BUILD_SHELL + is not set, then build bashInteractive from + . */ + auto shell = getEnv("NIX_BUILD_SHELL", ""); + + if (shell == "") { + + try { + auto expr = state->parseExprFromString("(import {}).bashInteractive", absPath(".")); + + Value v; + state->eval(expr, v); + + auto drv = getDerivation(*state, v, false); + if (!drv) + throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); + + pathsToBuild.insert(drv->queryDrvPath()); + + shell = drv->queryOutPath() + "/bin/bash"; + + } catch (Error & e) { + printError("warning: %s; will use bash from your environment", e.what()); + shell = "bash"; + } + } + + // Build or fetch all dependencies of the derivation. + for (const auto & input : drv.inputDrvs) + if (std::all_of(envExclude.cbegin(), envExclude.cend(), [&](const string & exclude) { return !std::regex_search(input.first, std::regex(exclude)); })) + pathsToBuild.insert(makeDrvPathWithOutputs(input.first, input.second)); + for (const auto & src : drv.inputSrcs) + pathsToBuild.insert(src); + + buildPaths(pathsToBuild); + + if (dryRun) return; + + // Set the environment. + auto env = getEnv(); + + auto tmp = getEnv("TMPDIR", getEnv("XDG_RUNTIME_DIR", "/tmp")); + + if (pure) { + decltype(env) newEnv; + for (auto & i : env) + if (keepVars.count(i.first)) + newEnv.emplace(i); + env = newEnv; + // NixOS hack: prevent /etc/bashrc from sourcing /etc/profile. + env["__ETC_PROFILE_SOURCED"] = "1"; + } + + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; + env["NIX_STORE"] = store->storeDir; + env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); + + auto passAsFile = tokenizeString(get(drv.env, "passAsFile", "")); + + bool keepTmp = false; + int fileNr = 0; + + for (auto & var : drv.env) + if (passAsFile.count(var.first)) { + keepTmp = true; + string fn = ".attr-" + std::to_string(fileNr++); + Path p = (Path) tmpDir + "/" + fn; + writeFile(p, var.second); + env[var.first + "Path"] = p; + } else + env[var.first] = var.second; + + restoreAffinity(); + + /* Run a shell using the derivation's environment. For + convenience, source $stdenv/setup to setup additional + environment variables and shell functions. Also don't + lose the current $PATH directories. */ + auto rcfile = (Path) tmpDir + "/rc"; + writeFile(rcfile, fmt( + (keepTmp ? "" : "rm -rf '%1%'; "s) + + "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc; " + "%2%" + "dontAddDisableDepTrack=1; " + "[ -e $stdenv/setup ] && source $stdenv/setup; " + "%3%" + "PATH=\"%4%:$PATH\"; " + "SHELL=%5%; " + "set +e; " + R"s([ -n "$PS1" ] && PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s" + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " + "unset NIX_ENFORCE_PURITY; " + "shopt -u nullglob; " + "unset TZ; %6%" + "%7%", + (Path) tmpDir, + (pure ? "" : "p=$PATH; "), + (pure ? "" : "PATH=$PATH:$p; unset p; "), + dirOf(shell), + shell, + (getenv("TZ") ? (string("export TZ='") + getenv("TZ") + "'; ") : ""), + envCommand)); + + Strings envStrs; + for (auto & i : env) + envStrs.push_back(i.first + "=" + i.second); + + auto args = interactive + ? Strings{"bash", "--rcfile", rcfile} + : Strings{"bash", rcfile}; + + auto envPtrs = stringsToCharPtrs(envStrs); + + environ = envPtrs.data(); + + auto argPtrs = stringsToCharPtrs(args); + + restoreSignals(); + + execvp(shell.c_str(), argPtrs.data()); + + throw SysError("executing shell '%s'", shell); + } + + else { + + PathSet pathsToBuild; + + std::map drvPrefixes; + std::map resultSymlinks; + std::vector outPaths; + + for (auto & drvInfo : drvs) { + auto drvPath = drvInfo.queryDrvPath(); + auto outPath = drvInfo.queryOutPath(); + + auto outputName = drvInfo.queryOutputName(); + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", drvPath); + + pathsToBuild.insert(drvPath + "!" + outputName); + + std::string drvPrefix; + auto i = drvPrefixes.find(drvPath); + if (i != drvPrefixes.end()) + drvPrefix = i->second; + else { + drvPrefix = outLink; + if (drvPrefixes.size()) + drvPrefix += fmt("-%d", drvPrefixes.size() + 1); + drvPrefixes[drvPath] = drvPrefix; + } + + std::string symlink = drvPrefix; + if (outputName != "out") symlink += "-" + outputName; + + resultSymlinks[symlink] = outPath; + outPaths.push_back(outPath); + } + + buildPaths(pathsToBuild); + + if (dryRun) return; + + for (auto & symlink : resultSymlinks) + if (auto store2 = store.dynamic_pointer_cast()) + store2->addPermRoot(symlink.second, absPath(symlink.first), true); + + for (auto & path : outPaths) + std::cout << path << '\n'; + } +} + +static RegisterLegacyCommand s1("nix-build", _main); +static RegisterLegacyCommand s2("nix-shell", _main); diff --git a/third_party/nix/src/nix-channel/nix-channel.cc b/third_party/nix/src/nix-channel/nix-channel.cc new file mode 100755 index 000000000..70aa5c966 --- /dev/null +++ b/third_party/nix/src/nix-channel/nix-channel.cc @@ -0,0 +1,248 @@ +#include "shared.hh" +#include "globals.hh" +#include "download.hh" +#include "store-api.hh" +#include "legacy.hh" + +#include +#include +#include + +using namespace nix; + +typedef std::map Channels; + +static Channels channels; +static Path channelsList; + +// Reads the list of channels. +static void readChannels() +{ + if (!pathExists(channelsList)) return; + auto channelsFile = readFile(channelsList); + + for (const auto & line : tokenizeString>(channelsFile, "\n")) { + chomp(line); + if (std::regex_search(line, std::regex("^\\s*\\#"))) + continue; + auto split = tokenizeString>(line, " "); + auto url = std::regex_replace(split[0], std::regex("/*$"), ""); + auto name = split.size() > 1 ? split[1] : baseNameOf(url); + channels[name] = url; + } +} + +// Writes the list of channels. +static void writeChannels() +{ + auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; + if (!channelsFD) + throw SysError(format("opening '%1%' for writing") % channelsList); + for (const auto & channel : channels) + writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); +} + +// Adds a channel. +static void addChannel(const string & url, const string & name) +{ + if (!regex_search(url, std::regex("^(file|http|https)://"))) + throw Error(format("invalid channel URL '%1%'") % url); + if (!regex_search(name, std::regex("^[a-zA-Z0-9_][a-zA-Z0-9_\\.-]*$"))) + throw Error(format("invalid channel identifier '%1%'") % name); + readChannels(); + channels[name] = url; + writeChannels(); +} + +static Path profile; + +// Remove a channel. +static void removeChannel(const string & name) +{ + readChannels(); + channels.erase(name); + writeChannels(); + + runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); +} + +static Path nixDefExpr; + +// Fetch Nix expressions and binary cache URLs from the subscribed channels. +static void update(const StringSet & channelNames) +{ + readChannels(); + + auto store = openStore(); + + // Download each channel. + Strings exprs; + for (const auto & channel : channels) { + auto name = channel.first; + auto url = channel.second; + if (!(channelNames.empty() || channelNames.count(name))) + continue; + + // We want to download the url to a file to see if it's a tarball while also checking if we + // got redirected in the process, so that we can grab the various parts of a nix channel + // definition from a consistent location if the redirect changes mid-download. + CachedDownloadRequest request(url); + request.ttl = 0; + auto dl = getDownloader(); + auto result = dl->downloadCached(store, request); + auto filename = result.path; + url = chomp(result.effectiveUri); + + // If the URL contains a version number, append it to the name + // attribute (so that "nix-env -q" on the channels profile + // shows something useful). + auto cname = name; + std::smatch match; + auto urlBase = baseNameOf(url); + if (std::regex_search(urlBase, match, std::regex("(-\\d.*)$"))) { + cname = cname + (string) match[1]; + } + + std::string extraAttrs; + + bool unpacked = false; + if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); + unpacked = true; + } + + if (!unpacked) { + // Download the channel tarball. + try { + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.xz")).path; + } catch (DownloadError & e) { + filename = dl->downloadCached(store, CachedDownloadRequest(url + "/nixexprs.tar.bz2")).path; + } + chomp(filename); + } + + // Regardless of where it came from, add the expression representing this channel to accumulated expression + exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); + } + + // Unpack the channel tarballs into the Nix store and install them + // into the channels profile. + std::cerr << "unpacking channels...\n"; + Strings envArgs{ "--profile", profile, "--file", "", "--install", "--from-expression" }; + for (auto & expr : exprs) + envArgs.push_back(std::move(expr)); + envArgs.push_back("--quiet"); + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + + // Make the channels appear in nix-env. + struct stat st; + if (lstat(nixDefExpr.c_str(), &st) == 0) { + if (S_ISLNK(st.st_mode)) + // old-skool ~/.nix-defexpr + if (unlink(nixDefExpr.c_str()) == -1) + throw SysError(format("unlinking %1%") % nixDefExpr); + } else if (errno != ENOENT) { + throw SysError(format("getting status of %1%") % nixDefExpr); + } + createDirs(nixDefExpr); + auto channelLink = nixDefExpr + "/channels"; + replaceSymlink(profile, channelLink); +} + +static int _main(int argc, char ** argv) +{ + { + // Figure out the name of the `.nix-channels' file to use + auto home = getHome(); + channelsList = home + "/.nix-channels"; + nixDefExpr = home + "/.nix-defexpr"; + + // Figure out the name of the channels profile. + profile = fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()); + + enum { + cNone, + cAdd, + cRemove, + cList, + cUpdate, + cRollback + } cmd = cNone; + std::vector args; + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") { + showManPage("nix-channel"); + } else if (*arg == "--version") { + printVersion("nix-channel"); + } else if (*arg == "--add") { + cmd = cAdd; + } else if (*arg == "--remove") { + cmd = cRemove; + } else if (*arg == "--list") { + cmd = cList; + } else if (*arg == "--update") { + cmd = cUpdate; + } else if (*arg == "--rollback") { + cmd = cRollback; + } else { + args.push_back(std::move(*arg)); + } + return true; + }); + + initPlugins(); + + switch (cmd) { + case cNone: + throw UsageError("no command specified"); + case cAdd: + if (args.size() < 1 || args.size() > 2) + throw UsageError("'--add' requires one or two arguments"); + { + auto url = args[0]; + std::string name; + if (args.size() == 2) { + name = args[1]; + } else { + name = baseNameOf(url); + name = std::regex_replace(name, std::regex("-unstable$"), ""); + name = std::regex_replace(name, std::regex("-stable$"), ""); + } + addChannel(url, name); + } + break; + case cRemove: + if (args.size() != 1) + throw UsageError("'--remove' requires one argument"); + removeChannel(args[0]); + break; + case cList: + if (!args.empty()) + throw UsageError("'--list' expects no arguments"); + readChannels(); + for (const auto & channel : channels) + std::cout << channel.first << ' ' << channel.second << '\n'; + break; + case cUpdate: + update(StringSet(args.begin(), args.end())); + break; + case cRollback: + if (args.size() > 1) + throw UsageError("'--rollback' has at most one argument"); + Strings envArgs{"--profile", profile}; + if (args.size() == 1) { + envArgs.push_back("--switch-generation"); + envArgs.push_back(args[0]); + } else { + envArgs.push_back("--rollback"); + } + runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + break; + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-channel", _main); diff --git a/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc b/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc new file mode 100644 index 000000000..d4060ac93 --- /dev/null +++ b/third_party/nix/src/nix-collect-garbage/nix-collect-garbage.cc @@ -0,0 +1,97 @@ +#include "store-api.hh" +#include "profiles.hh" +#include "shared.hh" +#include "globals.hh" +#include "legacy.hh" + +#include +#include + +using namespace nix; + +std::string deleteOlderThan; +bool dryRun = false; + + +/* If `-d' was specified, remove all old generations of all profiles. + * Of course, this makes rollbacks to before this point in time + * impossible. */ + +void removeOldGenerations(std::string dir) +{ + if (access(dir.c_str(), R_OK) != 0) return; + + bool canWrite = access(dir.c_str(), W_OK) == 0; + + for (auto & i : readDirectory(dir)) { + checkInterrupt(); + + auto path = dir + "/" + i.name; + auto type = i.type == DT_UNKNOWN ? getFileType(path) : i.type; + + if (type == DT_LNK && canWrite) { + std::string link; + try { + link = readLink(path); + } catch (SysError & e) { + if (e.errNo == ENOENT) continue; + } + if (link.find("link") != string::npos) { + printInfo(format("removing old generations of profile %1%") % path); + if (deleteOlderThan != "") + deleteGenerationsOlderThan(path, deleteOlderThan, dryRun); + else + deleteOldGenerations(path, dryRun); + } + } else if (type == DT_DIR) { + removeOldGenerations(path); + } + } +} + +static int _main(int argc, char * * argv) +{ + { + bool removeOld = false; + + GCOptions options; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-collect-garbage"); + else if (*arg == "--version") + printVersion("nix-collect-garbage"); + else if (*arg == "--delete-old" || *arg == "-d") removeOld = true; + else if (*arg == "--delete-older-than") { + removeOld = true; + deleteOlderThan = getArg(*arg, arg, end); + } + else if (*arg == "--dry-run") dryRun = true; + else if (*arg == "--max-freed") { + long long maxFreed = getIntArg(*arg, arg, end, true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } + else + return false; + return true; + }); + + initPlugins(); + + auto profilesDir = settings.nixStateDir + "/profiles"; + if (removeOld) removeOldGenerations(profilesDir); + + // Run the actual garbage collector. + if (!dryRun) { + auto store = openStore(); + options.action = GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-collect-garbage", _main); diff --git a/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc b/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc new file mode 100755 index 000000000..fdcde8b07 --- /dev/null +++ b/third_party/nix/src/nix-copy-closure/nix-copy-closure.cc @@ -0,0 +1,68 @@ +#include "shared.hh" +#include "store-api.hh" +#include "legacy.hh" + +using namespace nix; + +static int _main(int argc, char ** argv) +{ + { + auto gzip = false; + auto toMode = true; + auto includeOutputs = false; + auto dryRun = false; + auto useSubstitutes = NoSubstitute; + std::string sshHost; + PathSet storePaths; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-copy-closure"); + else if (*arg == "--version") + printVersion("nix-copy-closure"); + else if (*arg == "--gzip" || *arg == "--bzip2" || *arg == "--xz") { + if (*arg != "--gzip") + printMsg(lvlError, format("Warning: '%1%' is not implemented, falling back to gzip") % *arg); + gzip = true; + } else if (*arg == "--from") + toMode = false; + else if (*arg == "--to") + toMode = true; + else if (*arg == "--include-outputs") + includeOutputs = true; + else if (*arg == "--show-progress") + printMsg(lvlError, "Warning: '--show-progress' is not implemented"); + else if (*arg == "--dry-run") + dryRun = true; + else if (*arg == "--use-substitutes" || *arg == "-s") + useSubstitutes = Substitute; + else if (sshHost.empty()) + sshHost = *arg; + else + storePaths.insert(*arg); + return true; + }); + + initPlugins(); + + if (sshHost.empty()) + throw UsageError("no host name specified"); + + auto remoteUri = "ssh://" + sshHost + (gzip ? "?compress=true" : ""); + auto to = toMode ? openStore(remoteUri) : openStore(); + auto from = toMode ? openStore() : openStore(remoteUri); + + PathSet storePaths2; + for (auto & path : storePaths) + storePaths2.insert(from->followLinksToStorePath(path)); + + PathSet closure; + from->computeFSClosure(storePaths2, closure, false, includeOutputs); + + copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-copy-closure", _main); diff --git a/third_party/nix/src/nix-daemon/nix-daemon.cc b/third_party/nix/src/nix-daemon/nix-daemon.cc new file mode 100644 index 000000000..cd18489b0 --- /dev/null +++ b/third_party/nix/src/nix-daemon/nix-daemon.cc @@ -0,0 +1,1149 @@ +#include "shared.hh" +#include "local-store.hh" +#include "util.hh" +#include "serialise.hh" +#include "worker-protocol.hh" +#include "archive.hh" +#include "affinity.hh" +#include "globals.hh" +#include "monitor-fd.hh" +#include "derivations.hh" +#include "finally.hh" +#include "legacy.hh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if __APPLE__ || __FreeBSD__ +#include +#endif + +using namespace nix; + +#ifndef __linux__ +#define SPLICE_F_MOVE 0 +static ssize_t splice(int fd_in, void *off_in, int fd_out, void *off_out, size_t len, unsigned int flags) +{ + /* We ignore most parameters, we just have them for conformance with the linux syscall */ + std::vector buf(8192); + auto read_count = read(fd_in, buf.data(), buf.size()); + if (read_count == -1) + return read_count; + auto write_count = decltype(read_count)(0); + while (write_count < read_count) { + auto res = write(fd_out, buf.data() + write_count, read_count - write_count); + if (res == -1) + return res; + write_count += res; + } + return read_count; +} +#endif + +static FdSource from(STDIN_FILENO); +static FdSink to(STDOUT_FILENO); + + +Sink & operator << (Sink & sink, const Logger::Fields & fields) +{ + sink << fields.size(); + for (auto & f : fields) { + sink << f.type; + if (f.type == Logger::Field::tInt) + sink << f.i; + else if (f.type == Logger::Field::tString) + sink << f.s; + else abort(); + } + return sink; +} + + +/* Logger that forwards log messages to the client, *if* we're in a + state where the protocol allows it (i.e., when canSendStderr is + true). */ +struct TunnelLogger : public Logger +{ + struct State + { + bool canSendStderr = false; + std::vector pendingMsgs; + }; + + Sync state_; + + unsigned int clientVersion; + + TunnelLogger(unsigned int clientVersion) : clientVersion(clientVersion) { } + + void enqueueMsg(const std::string & s) + { + auto state(state_.lock()); + + if (state->canSendStderr) { + assert(state->pendingMsgs.empty()); + try { + to(s); + to.flush(); + } catch (...) { + /* Write failed; that means that the other side is + gone. */ + state->canSendStderr = false; + throw; + } + } else + state->pendingMsgs.push_back(s); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + if (lvl > verbosity) return; + + StringSink buf; + buf << STDERR_NEXT << (fs.s + "\n"); + enqueueMsg(*buf.s); + } + + /* startWork() means that we're starting an operation for which we + want to send out stderr to the client. */ + void startWork() + { + auto state(state_.lock()); + state->canSendStderr = true; + + for (auto & msg : state->pendingMsgs) + to(msg); + + state->pendingMsgs.clear(); + + to.flush(); + } + + /* stopWork() means that we're done; stop sending stderr to the + client. */ + void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) + { + auto state(state_.lock()); + + state->canSendStderr = false; + + if (success) + to << STDERR_LAST; + else { + to << STDERR_ERROR << msg; + if (status != 0) to << status; + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) { + if (!s.empty()) + log(lvl, s + "..."); + return; + } + + StringSink buf; + buf << STDERR_START_ACTIVITY << act << lvl << type << s << fields << parent; + enqueueMsg(*buf.s); + } + + void stopActivity(ActivityId act) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_STOP_ACTIVITY << act; + enqueueMsg(*buf.s); + } + + void result(ActivityId act, ResultType type, const Fields & fields) override + { + if (GET_PROTOCOL_MINOR(clientVersion) < 20) return; + StringSink buf; + buf << STDERR_RESULT << act << type << fields; + enqueueMsg(*buf.s); + } +}; + + +struct TunnelSink : Sink +{ + Sink & to; + TunnelSink(Sink & to) : to(to) { } + virtual void operator () (const unsigned char * data, size_t len) + { + to << STDERR_WRITE; + writeString(data, len, to); + } +}; + + +struct TunnelSource : BufferedSource +{ + Source & from; + TunnelSource(Source & from) : from(from) { } +protected: + size_t readUnbuffered(unsigned char * data, size_t len) override + { + to << STDERR_READ << len; + to.flush(); + size_t n = readString(data, len, from); + if (n == 0) throw EndOfFile("unexpected end-of-file"); + return n; + } +}; + + +/* If the NAR archive contains a single file at top-level, then save + the contents of the file to `s'. Otherwise barf. */ +struct RetrieveRegularNARSink : ParseSink +{ + bool regular; + string s; + + RetrieveRegularNARSink() : regular(true) { } + + void createDirectory(const Path & path) + { + regular = false; + } + + void receiveContents(unsigned char * data, unsigned int len) + { + s.append((const char *) data, len); + } + + void createSymlink(const Path & path, const string & target) + { + regular = false; + } +}; + + +static void performOp(TunnelLogger * logger, ref store, + bool trusted, unsigned int clientVersion, + Source & from, Sink & to, unsigned int op) +{ + switch (op) { + + case wopIsValidPath: { + /* 'readStorePath' could raise an error leading to the connection + being closed. To be able to recover from an invalid path error, + call 'startWork' early, and do 'assertStorePath' afterwards so + that the 'Error' exception handler doesn't close the + connection. */ + Path path = readString(from); + logger->startWork(); + store->assertStorePath(path); + bool result = store->isValidPath(path); + logger->stopWork(); + to << result; + break; + } + + case wopQueryValidPaths: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + PathSet res = store->queryValidPaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopHasSubstitutes: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths({path}); + logger->stopWork(); + to << (res.find(path) != res.end()); + break; + } + + case wopQuerySubstitutablePaths: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + PathSet res = store->querySubstitutablePaths(paths); + logger->stopWork(); + to << res; + break; + } + + case wopQueryPathHash: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto hash = store->queryPathInfo(path)->narHash; + logger->stopWork(); + to << hash.to_string(Base16, false); + break; + } + + case wopQueryReferences: + case wopQueryReferrers: + case wopQueryValidDerivers: + case wopQueryDerivationOutputs: { + Path path = readStorePath(*store, from); + logger->startWork(); + PathSet paths; + if (op == wopQueryReferences) + paths = store->queryPathInfo(path)->references; + else if (op == wopQueryReferrers) + store->queryReferrers(path, paths); + else if (op == wopQueryValidDerivers) + paths = store->queryValidDerivers(path); + else paths = store->queryDerivationOutputs(path); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryDerivationOutputNames: { + Path path = readStorePath(*store, from); + logger->startWork(); + StringSet names; + names = store->queryDerivationOutputNames(path); + logger->stopWork(); + to << names; + break; + } + + case wopQueryDeriver: { + Path path = readStorePath(*store, from); + logger->startWork(); + auto deriver = store->queryPathInfo(path)->deriver; + logger->stopWork(); + to << deriver; + break; + } + + case wopQueryPathFromHashPart: { + string hashPart = readString(from); + logger->startWork(); + Path path = store->queryPathFromHashPart(hashPart); + logger->stopWork(); + to << path; + break; + } + + case wopAddToStore: { + bool fixed, recursive; + std::string s, baseName; + from >> baseName >> fixed /* obsolete */ >> recursive >> s; + /* Compatibility hack. */ + if (!fixed) { + s = "sha256"; + recursive = true; + } + HashType hashAlgo = parseHashType(s); + + TeeSource savedNAR(from); + RetrieveRegularNARSink savedRegular; + + if (recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNAR); + } else + parseDump(savedRegular, from); + + logger->startWork(); + if (!savedRegular.regular) throw Error("regular file expected"); + + auto store2 = store.dynamic_pointer_cast(); + if (!store2) throw Error("operation is only supported by LocalStore"); + + Path path = store2->addToStoreFromDump(recursive ? *savedNAR.data : savedRegular.s, baseName, recursive, hashAlgo); + logger->stopWork(); + + to << path; + break; + } + + case wopAddTextToStore: { + string suffix = readString(from); + string s = readString(from); + PathSet refs = readStorePaths(*store, from); + logger->startWork(); + Path path = store->addTextToStore(suffix, s, refs, NoRepair); + logger->stopWork(); + to << path; + break; + } + + case wopExportPath: { + Path path = readStorePath(*store, from); + readInt(from); // obsolete + logger->startWork(); + TunnelSink sink(to); + store->exportPath(path, sink); + logger->stopWork(); + to << 1; + break; + } + + case wopImportPaths: { + logger->startWork(); + TunnelSource source(from); + Paths paths = store->importPaths(source, nullptr, + trusted ? NoCheckSigs : CheckSigs); + logger->stopWork(); + to << paths; + break; + } + + case wopBuildPaths: { + PathSet drvs = readStorePaths(*store, from); + BuildMode mode = bmNormal; + if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { + mode = (BuildMode) readInt(from); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); + } + logger->startWork(); + store->buildPaths(drvs, mode); + logger->stopWork(); + to << 1; + break; + } + + case wopBuildDerivation: { + Path drvPath = readStorePath(*store, from); + BasicDerivation drv; + readDerivation(from, *store, drv); + BuildMode buildMode = (BuildMode) readInt(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to build derivations"); + auto res = store->buildDerivation(drvPath, drv, buildMode); + logger->stopWork(); + to << res.status << res.errorMsg; + break; + } + + case wopEnsurePath: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->ensurePath(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddTempRoot: { + Path path = readStorePath(*store, from); + logger->startWork(); + store->addTempRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopAddIndirectRoot: { + Path path = absPath(readString(from)); + logger->startWork(); + store->addIndirectRoot(path); + logger->stopWork(); + to << 1; + break; + } + + case wopSyncWithGC: { + logger->startWork(); + store->syncWithGC(); + logger->stopWork(); + to << 1; + break; + } + + case wopFindRoots: { + logger->startWork(); + Roots roots = store->findRoots(!trusted); + logger->stopWork(); + + size_t size = 0; + for (auto & i : roots) + size += i.second.size(); + + to << size; + + for (auto & [target, links] : roots) + for (auto & link : links) + to << link << target; + + break; + } + + case wopCollectGarbage: { + GCOptions options; + options.action = (GCOptions::GCAction) readInt(from); + options.pathsToDelete = readStorePaths(*store, from); + from >> options.ignoreLiveness >> options.maxFreed; + // obsolete fields + readInt(from); + readInt(from); + readInt(from); + + GCResults results; + + logger->startWork(); + if (options.ignoreLiveness) + throw Error("you are not allowed to ignore liveness"); + store->collectGarbage(options, results); + logger->stopWork(); + + to << results.paths << results.bytesFreed << 0 /* obsolete */; + + break; + } + + case wopSetOptions: { + settings.keepFailed = readInt(from); + settings.keepGoing = readInt(from); + settings.tryFallback = readInt(from); + verbosity = (Verbosity) readInt(from); + settings.maxBuildJobs.assign(readInt(from)); + settings.maxSilentTime = readInt(from); + readInt(from); // obsolete useBuildHook + settings.verboseBuild = lvlError == (Verbosity) readInt(from); + readInt(from); // obsolete logType + readInt(from); // obsolete printBuildTrace + settings.buildCores = readInt(from); + settings.useSubstitutes = readInt(from); + + StringMap overrides; + if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { + unsigned int n = readInt(from); + for (unsigned int i = 0; i < n; i++) { + string name = readString(from); + string value = readString(from); + overrides.emplace(name, value); + } + } + + logger->startWork(); + + for (auto & i : overrides) { + auto & name(i.first); + auto & value(i.second); + + auto setSubstituters = [&](Setting & res) { + if (name != res.name && res.aliases.count(name) == 0) + return false; + StringSet trusted = settings.trustedSubstituters; + for (auto & s : settings.substituters.get()) + trusted.insert(s); + Strings subs; + auto ss = tokenizeString(value); + for (auto & s : ss) + if (trusted.count(s)) + subs.push_back(s); + else + warn("ignoring untrusted substituter '%s'", s); + res = subs; + return true; + }; + + try { + if (name == "ssh-auth-sock") // obsolete + ; + else if (trusted + || name == settings.buildTimeout.name + || name == "connect-timeout" + || (name == "builders" && value == "")) + settings.set(name, value); + else if (setSubstituters(settings.substituters)) + ; + else if (setSubstituters(settings.extraSubstituters)) + ; + else + warn("ignoring the user-specified setting '%s', because it is a restricted setting and you are not a trusted user", name); + } catch (UsageError & e) { + warn(e.what()); + } + } + + logger->stopWork(); + break; + } + + case wopQuerySubstitutablePathInfo: { + Path path = absPath(readString(from)); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos({path}, infos); + logger->stopWork(); + SubstitutablePathInfos::iterator i = infos.find(path); + if (i == infos.end()) + to << 0; + else { + to << 1 << i->second.deriver << i->second.references << i->second.downloadSize << i->second.narSize; + } + break; + } + + case wopQuerySubstitutablePathInfos: { + PathSet paths = readStorePaths(*store, from); + logger->startWork(); + SubstitutablePathInfos infos; + store->querySubstitutablePathInfos(paths, infos); + logger->stopWork(); + to << infos.size(); + for (auto & i : infos) { + to << i.first << i.second.deriver << i.second.references + << i.second.downloadSize << i.second.narSize; + } + break; + } + + case wopQueryAllValidPaths: { + logger->startWork(); + PathSet paths = store->queryAllValidPaths(); + logger->stopWork(); + to << paths; + break; + } + + case wopQueryPathInfo: { + Path path = readStorePath(*store, from); + std::shared_ptr info; + logger->startWork(); + try { + info = store->queryPathInfo(path); + } catch (InvalidPath &) { + if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + } + logger->stopWork(); + if (info) { + if (GET_PROTOCOL_MINOR(clientVersion) >= 17) + to << 1; + to << info->deriver << info->narHash.to_string(Base16, false) << info->references + << info->registrationTime << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 16) { + to << info->ultimate + << info->sigs + << info->ca; + } + } else { + assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); + to << 0; + } + break; + } + + case wopOptimiseStore: + logger->startWork(); + store->optimiseStore(); + logger->stopWork(); + to << 1; + break; + + case wopVerifyStore: { + bool checkContents, repair; + from >> checkContents >> repair; + logger->startWork(); + if (repair && !trusted) + throw Error("you are not privileged to repair paths"); + bool errors = store->verifyStore(checkContents, (RepairFlag) repair); + logger->stopWork(); + to << errors; + break; + } + + case wopAddSignatures: { + Path path = readStorePath(*store, from); + StringSet sigs = readStrings(from); + logger->startWork(); + if (!trusted) + throw Error("you are not privileged to add signatures"); + store->addSignatures(path, sigs); + logger->stopWork(); + to << 1; + break; + } + + case wopNarFromPath: { + auto path = readStorePath(*store, from); + logger->startWork(); + logger->stopWork(); + dumpPath(path, to); + break; + } + + case wopAddToStoreNar: { + bool repair, dontCheckSigs; + ValidPathInfo info; + info.path = readStorePath(*store, from); + from >> info.deriver; + if (!info.deriver.empty()) + store->assertStorePath(info.deriver); + info.narHash = Hash(readString(from), htSHA256); + info.references = readStorePaths(*store, from); + from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(from); + from >> info.ca >> repair >> dontCheckSigs; + if (!trusted && dontCheckSigs) + dontCheckSigs = false; + if (!trusted) + info.ultimate = false; + + std::string saved; + std::unique_ptr source; + if (GET_PROTOCOL_MINOR(clientVersion) >= 21) + source = std::make_unique(from); + else { + TeeSink tee(from); + parseDump(tee, tee.source); + saved = std::move(*tee.source.data); + source = std::make_unique(saved); + } + + logger->startWork(); + + // FIXME: race if addToStore doesn't read source? + store->addToStore(info, *source, (RepairFlag) repair, + dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr); + + logger->stopWork(); + break; + } + + case wopQueryMissing: { + PathSet targets = readStorePaths(*store, from); + logger->startWork(); + PathSet willBuild, willSubstitute, unknown; + unsigned long long downloadSize, narSize; + store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize); + logger->stopWork(); + to << willBuild << willSubstitute << unknown << downloadSize << narSize; + break; + } + + default: + throw Error(format("invalid operation %1%") % op); + } +} + + +static void processConnection(bool trusted, + const std::string & userName, uid_t userId) +{ + MonitorFdHup monitor(from.fd); + + /* Exchange the greeting. */ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << PROTOCOL_VERSION; + to.flush(); + unsigned int clientVersion = readInt(from); + + if (clientVersion < 0x10a) + throw Error("the Nix client version is too old"); + + auto tunnelLogger = new TunnelLogger(clientVersion); + auto prevLogger = nix::logger; + logger = tunnelLogger; + + unsigned int opCount = 0; + + Finally finally([&]() { + _isInterrupted = false; + prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + }); + + if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) + setAffinityTo(readInt(from)); + + readInt(from); // obsolete reserveSpace + + /* Send startup error messages to the client. */ + tunnelLogger->startWork(); + + try { + + /* If we can't accept clientVersion, then throw an error + *here* (not above). */ + +#if 0 + /* Prevent users from doing something very dangerous. */ + if (geteuid() == 0 && + querySetting("build-users-group", "") == "") + throw Error("if you run 'nix-daemon' as root, then you MUST set 'build-users-group'!"); +#endif + + /* Open the store. */ + Store::Params params; // FIXME: get params from somewhere + // Disable caching since the client already does that. + params["path-info-cache-size"] = "0"; + auto store = openStore(settings.storeUri, params); + + store->createUser(userName, userId); + + tunnelLogger->stopWork(); + to.flush(); + + /* Process client requests. */ + while (true) { + WorkerOp op; + try { + op = (WorkerOp) readInt(from); + } catch (Interrupted & e) { + break; + } catch (EndOfFile & e) { + break; + } + + opCount++; + + try { + performOp(tunnelLogger, store, trusted, clientVersion, from, to, op); + } catch (Error & e) { + /* If we're not in a state where we can send replies, then + something went wrong processing the input of the + client. This can happen especially if I/O errors occur + during addTextToStore() / importPath(). If that + happens, just send the error message and exit. */ + bool errorAllowed = tunnelLogger->state_.lock()->canSendStderr; + tunnelLogger->stopWork(false, e.msg(), e.status); + if (!errorAllowed) throw; + } catch (std::bad_alloc & e) { + tunnelLogger->stopWork(false, "Nix daemon out of memory", 1); + throw; + } + + to.flush(); + + assert(!tunnelLogger->state_.lock()->canSendStderr); + }; + + } catch (std::exception & e) { + tunnelLogger->stopWork(false, e.what(), 1); + to.flush(); + return; + } +} + + +static void sigChldHandler(int sigNo) +{ + // Ensure we don't modify errno of whatever we've interrupted + auto saved_errno = errno; + /* Reap all dead children. */ + while (waitpid(-1, 0, WNOHANG) > 0) ; + errno = saved_errno; +} + + +static void setSigChldAction(bool autoReap) +{ + struct sigaction act, oact; + act.sa_handler = autoReap ? sigChldHandler : SIG_DFL; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGCHLD, &act, &oact)) + throw SysError("setting SIGCHLD handler"); +} + + +bool matchUser(const string & user, const string & group, const Strings & users) +{ + if (find(users.begin(), users.end(), "*") != users.end()) + return true; + + if (find(users.begin(), users.end(), user) != users.end()) + return true; + + for (auto & i : users) + if (string(i, 0, 1) == "@") { + if (group == string(i, 1)) return true; + struct group * gr = getgrnam(i.c_str() + 1); + if (!gr) continue; + for (char * * mem = gr->gr_mem; *mem; mem++) + if (user == string(*mem)) return true; + } + + return false; +} + + +struct PeerInfo +{ + bool pidKnown; + pid_t pid; + bool uidKnown; + uid_t uid; + bool gidKnown; + gid_t gid; +}; + + +/* Get the identity of the caller, if possible. */ +static PeerInfo getPeerInfo(int remote) +{ + PeerInfo peer = { false, 0, false, 0, false, 0 }; + +#if defined(SO_PEERCRED) + + ucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { true, cred.pid, true, cred.uid, true, cred.gid }; + +#elif defined(LOCAL_PEERCRED) + +#if !defined(SOL_LOCAL) +#define SOL_LOCAL 0 +#endif + + xucred cred; + socklen_t credLen = sizeof(cred); + if (getsockopt(remote, SOL_LOCAL, LOCAL_PEERCRED, &cred, &credLen) == -1) + throw SysError("getting peer credentials"); + peer = { false, 0, true, cred.cr_uid, false, 0 }; + +#endif + + return peer; +} + + +#define SD_LISTEN_FDS_START 3 + + +static void daemonLoop(char * * argv) +{ + if (chdir("/") == -1) + throw SysError("cannot change current directory"); + + /* Get rid of children automatically; don't let them become + zombies. */ + setSigChldAction(true); + + AutoCloseFD fdSocket; + + /* Handle socket-based activation by systemd. */ + if (getEnv("LISTEN_FDS") != "") { + if (getEnv("LISTEN_PID") != std::to_string(getpid()) || getEnv("LISTEN_FDS") != "1") + throw Error("unexpected systemd environment variables"); + fdSocket = SD_LISTEN_FDS_START; + } + + /* Otherwise, create and bind to a Unix domain socket. */ + else { + + /* Create and bind to a Unix domain socket. */ + fdSocket = socket(PF_UNIX, SOCK_STREAM, 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + + string socketPath = settings.nixDaemonSocketFile; + + createDirs(dirOf(socketPath)); + + /* Urgh, sockaddr_un allows path names of only 108 characters. + So chdir to the socket directory so that we can pass a + relative path name. */ + if (chdir(dirOf(socketPath).c_str()) == -1) + throw SysError("cannot change current directory"); + Path socketPathRel = "./" + baseNameOf(socketPath); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + if (socketPathRel.size() >= sizeof(addr.sun_path)) + throw Error(format("socket path '%1%' is too long") % socketPathRel); + strcpy(addr.sun_path, socketPathRel.c_str()); + + unlink(socketPath.c_str()); + + /* Make sure that the socket is created with 0666 permission + (everybody can connect --- provided they have access to the + directory containing the socket). */ + mode_t oldMode = umask(0111); + int res = bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)); + umask(oldMode); + if (res == -1) + throw SysError(format("cannot bind to socket '%1%'") % socketPath); + + if (chdir("/") == -1) /* back to the root */ + throw SysError("cannot change current directory"); + + if (listen(fdSocket.get(), 5) == -1) + throw SysError(format("cannot listen on socket '%1%'") % socketPath); + } + + closeOnExec(fdSocket.get()); + + /* Loop accepting connections. */ + while (1) { + + try { + /* Accept a connection. */ + struct sockaddr_un remoteAddr; + socklen_t remoteAddrLen = sizeof(remoteAddr); + + AutoCloseFD remote = accept(fdSocket.get(), + (struct sockaddr *) &remoteAddr, &remoteAddrLen); + checkInterrupt(); + if (!remote) { + if (errno == EINTR) continue; + throw SysError("accepting connection"); + } + + closeOnExec(remote.get()); + + bool trusted = false; + PeerInfo peer = getPeerInfo(remote.get()); + + struct passwd * pw = peer.uidKnown ? getpwuid(peer.uid) : 0; + string user = pw ? pw->pw_name : std::to_string(peer.uid); + + struct group * gr = peer.gidKnown ? getgrgid(peer.gid) : 0; + string group = gr ? gr->gr_name : std::to_string(peer.gid); + + Strings trustedUsers = settings.trustedUsers; + Strings allowedUsers = settings.allowedUsers; + + if (matchUser(user, group, trustedUsers)) + trusted = true; + + if ((!trusted && !matchUser(user, group, allowedUsers)) || group == settings.buildUsersGroup) + throw Error(format("user '%1%' is not allowed to connect to the Nix daemon") % user); + + printInfo(format((string) "accepted connection from pid %1%, user %2%" + (trusted ? " (trusted)" : "")) + % (peer.pidKnown ? std::to_string(peer.pid) : "") + % (peer.uidKnown ? user : "")); + + /* Fork a child to handle the connection. */ + ProcessOptions options; + options.errorPrefix = "unexpected Nix daemon error: "; + options.dieWithParent = false; + options.runExitHandlers = true; + options.allowVfork = false; + startProcess([&]() { + fdSocket = -1; + + /* Background the daemon. */ + if (setsid() == -1) + throw SysError(format("creating a new session")); + + /* Restore normal handling of SIGCHLD. */ + setSigChldAction(false); + + /* For debugging, stuff the pid into argv[1]. */ + if (peer.pidKnown && argv[1]) { + string processName = std::to_string(peer.pid); + strncpy(argv[1], processName.c_str(), strlen(argv[1])); + } + + /* Handle the connection. */ + from.fd = remote.get(); + to.fd = remote.get(); + processConnection(trusted, user, peer.uid); + + exit(0); + }, options); + + } catch (Interrupted & e) { + return; + } catch (Error & e) { + printError(format("error processing connection: %1%") % e.msg()); + } + } +} + + +static int _main(int argc, char * * argv) +{ + { + auto stdio = false; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--daemon") + ; /* ignored for backwards compatibility */ + else if (*arg == "--help") + showManPage("nix-daemon"); + else if (*arg == "--version") + printVersion("nix-daemon"); + else if (*arg == "--stdio") + stdio = true; + else return false; + return true; + }); + + initPlugins(); + + if (stdio) { + if (getStoreType() == tDaemon) { + /* Forward on this connection to the real daemon */ + auto socketPath = settings.nixDaemonSocketFile; + auto s = socket(PF_UNIX, SOCK_STREAM, 0); + if (s == -1) + throw SysError("creating Unix domain socket"); + + auto socketDir = dirOf(socketPath); + if (chdir(socketDir.c_str()) == -1) + throw SysError(format("changing to socket directory '%1%'") % socketDir); + + auto socketName = baseNameOf(socketPath); + auto addr = sockaddr_un{}; + addr.sun_family = AF_UNIX; + if (socketName.size() + 1 >= sizeof(addr.sun_path)) + throw Error(format("socket name %1% is too long") % socketName); + strcpy(addr.sun_path, socketName.c_str()); + + if (connect(s, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError(format("cannot connect to daemon at %1%") % socketPath); + + auto nfds = (s > STDIN_FILENO ? s : STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(s, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(s, &fds)) { + auto res = splice(s, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, s, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return 0; + } + } + } else { + processConnection(true, "root", 0); + } + } else { + daemonLoop(argv); + } + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-daemon", _main); diff --git a/third_party/nix/src/nix-env/nix-env.cc b/third_party/nix/src/nix-env/nix-env.cc new file mode 100644 index 000000000..3292f546a --- /dev/null +++ b/third_party/nix/src/nix-env/nix-env.cc @@ -0,0 +1,1457 @@ +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "globals.hh" +#include "names.hh" +#include "profiles.hh" +#include "shared.hh" +#include "store-api.hh" +#include "user-env.hh" +#include "util.hh" +#include "json.hh" +#include "value-to-json.hh" +#include "xml-writer.hh" +#include "legacy.hh" + +#include +#include +#include +#include +#include + +#include +#include +#include + + +using namespace nix; +using std::cout; + + +typedef enum { + srcNixExprDrvs, + srcNixExprs, + srcStorePaths, + srcProfile, + srcAttrPath, + srcUnknown +} InstallSourceType; + + +struct InstallSourceInfo +{ + InstallSourceType type; + Path nixExprPath; /* for srcNixExprDrvs, srcNixExprs */ + Path profile; /* for srcProfile */ + string systemFilter; /* for srcNixExprDrvs */ + Bindings * autoArgs; +}; + + +struct Globals +{ + InstallSourceInfo instSource; + Path profile; + std::shared_ptr state; + bool dryRun; + bool preserveInstalled; + bool removeAll; + string forceName; + bool prebuiltOnly; +}; + + +typedef void (* Operation) (Globals & globals, + Strings opFlags, Strings opArgs); + + +static string needArg(Strings::iterator & i, + Strings & args, const string & arg) +{ + if (i == args.end()) throw UsageError( + format("'%1%' requires an argument") % arg); + return *i++; +} + + +static bool parseInstallSourceOptions(Globals & globals, + Strings::iterator & i, Strings & args, const string & arg) +{ + if (arg == "--from-expression" || arg == "-E") + globals.instSource.type = srcNixExprs; + else if (arg == "--from-profile") { + globals.instSource.type = srcProfile; + globals.instSource.profile = needArg(i, args, arg); + } + else if (arg == "--attr" || arg == "-A") + globals.instSource.type = srcAttrPath; + else return false; + return true; +} + + +static bool isNixExpr(const Path & path, struct stat & st) +{ + return S_ISREG(st.st_mode) || (S_ISDIR(st.st_mode) && pathExists(path + "/default.nix")); +} + + +static void getAllExprs(EvalState & state, + const Path & path, StringSet & attrs, Value & v) +{ + StringSet namesSorted; + for (auto & i : readDirectory(path)) namesSorted.insert(i.name); + + for (auto & i : namesSorted) { + /* Ignore the manifest.nix used by profiles. This is + necessary to prevent it from showing up in channels (which + are implemented using profiles). */ + if (i == "manifest.nix") continue; + + Path path2 = path + "/" + i; + + struct stat st; + if (stat(path2.c_str(), &st) == -1) + continue; // ignore dangling symlinks in ~/.nix-defexpr + + if (isNixExpr(path2, st) && (!S_ISREG(st.st_mode) || hasSuffix(path2, ".nix"))) { + /* Strip off the `.nix' filename suffix (if applicable), + otherwise the attribute cannot be selected with the + `-A' option. Useful if you want to stick a Nix + expression directly in ~/.nix-defexpr. */ + string attrName = i; + if (hasSuffix(attrName, ".nix")) + attrName = string(attrName, 0, attrName.size() - 4); + if (attrs.find(attrName) != attrs.end()) { + printError(format("warning: name collision in input Nix expressions, skipping '%1%'") % path2); + continue; + } + attrs.insert(attrName); + /* Load the expression on demand. */ + Value & vFun = state.getBuiltin("import"); + Value & vArg(*state.allocValue()); + mkString(vArg, path2); + if (v.attrs->size() == v.attrs->capacity()) + throw Error(format("too many Nix expressions in directory '%1%'") % path); + mkApp(*state.allocAttr(v, state.symbols.create(attrName)), vFun, vArg); + } + else if (S_ISDIR(st.st_mode)) + /* `path2' is a directory (with no default.nix in it); + recurse into it. */ + getAllExprs(state, path2, attrs, v); + } +} + + +static void loadSourceExpr(EvalState & state, const Path & path, Value & v) +{ + struct stat st; + if (stat(path.c_str(), &st) == -1) + throw SysError(format("getting information about '%1%'") % path); + + if (isNixExpr(path, st)) + state.evalFile(path, v); + + /* The path is a directory. Put the Nix expressions in the + directory in a set, with the file name of each expression as + the attribute name. Recurse into subdirectories (but keep the + set flat, not nested, to make it easier for a user to have a + ~/.nix-defexpr directory that includes some system-wide + directory). */ + else if (S_ISDIR(st.st_mode)) { + state.mkAttrs(v, 1024); + state.mkList(*state.allocAttr(v, state.symbols.create("_combineChannels")), 0); + StringSet attrs; + getAllExprs(state, path, attrs, v); + v.attrs->sort(); + } + + else throw Error("path '%s' is not a directory or a Nix expression", path); +} + + +static void loadDerivations(EvalState & state, Path nixExprPath, + string systemFilter, Bindings & autoArgs, + const string & pathPrefix, DrvInfos & elems) +{ + Value vRoot; + loadSourceExpr(state, nixExprPath, vRoot); + + Value & v(*findAlongAttrPath(state, pathPrefix, autoArgs, vRoot)); + + getDerivations(state, v, pathPrefix, autoArgs, elems, true); + + /* Filter out all derivations not applicable to the current + system. */ + for (DrvInfos::iterator i = elems.begin(), j; i != elems.end(); i = j) { + j = i; j++; + if (systemFilter != "*" && i->querySystem() != systemFilter) + elems.erase(i); + } +} + + +static long getPriority(EvalState & state, DrvInfo & drv) +{ + return drv.queryMetaInt("priority", 0); +} + + +static long comparePriorities(EvalState & state, DrvInfo & drv1, DrvInfo & drv2) +{ + return getPriority(state, drv2) - getPriority(state, drv1); +} + + +// FIXME: this function is rather slow since it checks a single path +// at a time. +static bool isPrebuilt(EvalState & state, DrvInfo & elem) +{ + Path path = elem.queryOutPath(); + if (state.store->isValidPath(path)) return true; + PathSet ps = state.store->querySubstitutablePaths({path}); + return ps.find(path) != ps.end(); +} + + +static void checkSelectorUse(DrvNames & selectors) +{ + /* Check that all selectors have been used. */ + for (auto & i : selectors) + if (i.hits == 0 && i.fullName != "*") + throw Error(format("selector '%1%' matches no derivations") % i.fullName); +} + + +static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, + const Strings & args, bool newestOnly) +{ + DrvNames selectors = drvNamesFromArgs(args); + if (selectors.empty()) + selectors.push_back(DrvName("*")); + + DrvInfos elems; + set done; + + for (auto & i : selectors) { + typedef list > Matches; + Matches matches; + unsigned int n = 0; + for (DrvInfos::const_iterator j = allElems.begin(); + j != allElems.end(); ++j, ++n) + { + DrvName drvName(j->queryName()); + if (i.matches(drvName)) { + i.hits++; + matches.push_back(std::pair(*j, n)); + } + } + + /* If `newestOnly', if a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ + if (newestOnly) { + + /* Map from package names to derivations. */ + typedef map > Newest; + Newest newest; + StringSet multiple; + + for (auto & j : matches) { + DrvName drvName(j.first.queryName()); + long d = 1; + + Newest::iterator k = newest.find(drvName.name); + + if (k != newest.end()) { + d = j.first.querySystem() == k->second.first.querySystem() ? 0 : + j.first.querySystem() == settings.thisSystem ? 1 : + k->second.first.querySystem() == settings.thisSystem ? -1 : 0; + if (d == 0) + d = comparePriorities(state, j.first, k->second.first); + if (d == 0) + d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); + } + + if (d > 0) { + newest.erase(drvName.name); + newest.insert(Newest::value_type(drvName.name, j)); + multiple.erase(j.first.queryName()); + } else if (d == 0) { + multiple.insert(j.first.queryName()); + } + } + + matches.clear(); + for (auto & j : newest) { + if (multiple.find(j.second.first.queryName()) != multiple.end()) + printInfo( + "warning: there are multiple derivations named '%1%'; using the first one", + j.second.first.queryName()); + matches.push_back(j.second); + } + } + + /* Insert only those elements in the final list that we + haven't inserted before. */ + for (auto & j : matches) + if (done.find(j.second) == done.end()) { + done.insert(j.second); + elems.push_back(j.first); + } + } + + checkSelectorUse(selectors); + + return elems; +} + + +static bool isPath(const string & s) +{ + return s.find('/') != string::npos; +} + + +static void queryInstSources(EvalState & state, + InstallSourceInfo & instSource, const Strings & args, + DrvInfos & elems, bool newestOnly) +{ + InstallSourceType type = instSource.type; + if (type == srcUnknown && args.size() > 0 && isPath(args.front())) + type = srcStorePaths; + + switch (type) { + + /* Get the available user environment elements from the + derivations specified in a Nix expression, including only + those with names matching any of the names in `args'. */ + case srcUnknown: + case srcNixExprDrvs: { + + /* Load the derivations from the (default or specified) + Nix expression. */ + DrvInfos allElems; + loadDerivations(state, instSource.nixExprPath, + instSource.systemFilter, *instSource.autoArgs, "", allElems); + + elems = filterBySelector(state, allElems, args, newestOnly); + + break; + } + + /* Get the available user environment elements from the Nix + expressions specified on the command line; these should be + functions that take the default Nix expression file as + argument, e.g., if the file is `./foo.nix', then the + argument `x: x.bar' is equivalent to `(x: x.bar) + (import ./foo.nix)' = `(import ./foo.nix).bar'. */ + case srcNixExprs: { + + Value vArg; + loadSourceExpr(state, instSource.nixExprPath, vArg); + + for (auto & i : args) { + Expr * eFun = state.parseExprFromString(i, absPath(".")); + Value vFun, vTmp; + state.eval(eFun, vFun); + mkApp(vTmp, vFun, vArg); + getDerivations(state, vTmp, "", *instSource.autoArgs, elems, true); + } + + break; + } + + /* The available user environment elements are specified as a + list of store paths (which may or may not be + derivations). */ + case srcStorePaths: { + + for (auto & i : args) { + Path path = state.store->followLinksToStorePath(i); + + string name = baseNameOf(path); + string::size_type dash = name.find('-'); + if (dash != string::npos) + name = string(name, dash + 1); + + DrvInfo elem(state, "", nullptr); + elem.setName(name); + + if (isDerivation(path)) { + elem.setDrvPath(path); + elem.setOutPath(state.store->derivationFromPath(path).findOutput("out")); + if (name.size() >= drvExtension.size() && + string(name, name.size() - drvExtension.size()) == drvExtension) + name = string(name, 0, name.size() - drvExtension.size()); + } + else elem.setOutPath(path); + + elems.push_back(elem); + } + + break; + } + + /* Get the available user environment elements from another + user environment. These are then filtered as in the + `srcNixExprDrvs' case. */ + case srcProfile: { + elems = filterBySelector(state, + queryInstalled(state, instSource.profile), + args, newestOnly); + break; + } + + case srcAttrPath: { + Value vRoot; + loadSourceExpr(state, instSource.nixExprPath, vRoot); + for (auto & i : args) { + Value & v(*findAlongAttrPath(state, i, *instSource.autoArgs, vRoot)); + getDerivations(state, v, "", *instSource.autoArgs, elems, true); + } + break; + } + } +} + + +static void printMissing(EvalState & state, DrvInfos & elems) +{ + PathSet targets; + for (auto & i : elems) { + Path drvPath = i.queryDrvPath(); + if (drvPath != "") + targets.insert(drvPath); + else + targets.insert(i.queryOutPath()); + } + + printMissing(state.store, targets); +} + + +static bool keep(DrvInfo & drv) +{ + return drv.queryMetaBool("keep", false); +} + + +static void installDerivations(Globals & globals, + const Strings & args, const Path & profile) +{ + debug(format("installing derivations")); + + /* Get the set of user environment elements to be installed. */ + DrvInfos newElems, newElemsTmp; + queryInstSources(*globals.state, globals.instSource, args, newElemsTmp, true); + + /* If --prebuilt-only is given, filter out source-only packages. */ + for (auto & i : newElemsTmp) + if (!globals.prebuiltOnly || isPrebuilt(*globals.state, i)) + newElems.push_back(i); + + StringSet newNames; + for (auto & i : newElems) { + /* `forceName' is a hack to get package names right in some + one-click installs, namely those where the name used in the + path is not the one we want (e.g., `java-front' versus + `java-front-0.9pre15899'). */ + if (globals.forceName != "") + i.setName(globals.forceName); + newNames.insert(DrvName(i.queryName()).name); + } + + + while (true) { + string lockToken = optimisticLockProfile(profile); + + DrvInfos allElems(newElems); + + /* Add in the already installed derivations, unless they have + the same name as a to-be-installed element. */ + if (!globals.removeAll) { + DrvInfos installedElems = queryInstalled(*globals.state, profile); + + for (auto & i : installedElems) { + DrvName drvName(i.queryName()); + if (!globals.preserveInstalled && + newNames.find(drvName.name) != newNames.end() && + !keep(i)) + printInfo("replacing old '%s'", i.queryName()); + else + allElems.push_back(i); + } + + for (auto & i : newElems) + printInfo("installing '%s'", i.queryName()); + } + + printMissing(*globals.state, newElems); + + if (globals.dryRun) return; + + if (createUserEnv(*globals.state, allElems, + profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opInstall(Globals & globals, Strings opFlags, Strings opArgs) +{ + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; + else if (arg == "--preserve-installed" || arg == "-P") + globals.preserveInstalled = true; + else if (arg == "--remove-all" || arg == "-r") + globals.removeAll = true; + else throw UsageError(format("unknown flag '%1%'") % arg); + } + + installDerivations(globals, opArgs, globals.profile); +} + + +typedef enum { utLt, utLeq, utEq, utAlways } UpgradeType; + + +static void upgradeDerivations(Globals & globals, + const Strings & args, UpgradeType upgradeType) +{ + debug(format("upgrading derivations")); + + /* Upgrade works as follows: we take all currently installed + derivations, and for any derivation matching any selector, look + for a derivation in the input Nix expression that has the same + name and a higher version number. */ + + while (true) { + string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + + /* Fetch all derivations from the input file. */ + DrvInfos availElems; + queryInstSources(*globals.state, globals.instSource, args, availElems, false); + + /* Go through all installed derivations. */ + DrvInfos newElems; + for (auto & i : installedElems) { + DrvName drvName(i.queryName()); + + try { + + if (keep(i)) { + newElems.push_back(i); + continue; + } + + /* Find the derivation in the input Nix expression + with the same name that satisfies the version + constraints specified by upgradeType. If there are + multiple matches, take the one with the highest + priority. If there are still multiple matches, + take the one with the highest version. + Do not upgrade if it would decrease the priority. */ + DrvInfos::iterator bestElem = availElems.end(); + string bestVersion; + for (auto j = availElems.begin(); j != availElems.end(); ++j) { + if (comparePriorities(*globals.state, i, *j) > 0) + continue; + DrvName newName(j->queryName()); + if (newName.name == drvName.name) { + int d = compareVersions(drvName.version, newName.version); + if ((upgradeType == utLt && d < 0) || + (upgradeType == utLeq && d <= 0) || + (upgradeType == utEq && d == 0) || + upgradeType == utAlways) + { + long d2 = -1; + if (bestElem != availElems.end()) { + d2 = comparePriorities(*globals.state, *bestElem, *j); + if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); + } + if (d2 < 0 && (!globals.prebuiltOnly || isPrebuilt(*globals.state, *j))) { + bestElem = j; + bestVersion = newName.version; + } + } + } + } + + if (bestElem != availElems.end() && + i.queryOutPath() != + bestElem->queryOutPath()) + { + const char * action = compareVersions(drvName.version, bestVersion) <= 0 + ? "upgrading" : "downgrading"; + printInfo("%1% '%2%' to '%3%'", + action, i.queryName(), bestElem->queryName()); + newElems.push_back(*bestElem); + } else newElems.push_back(i); + + } catch (Error & e) { + e.addPrefix(fmt("while trying to find an upgrade for '%s':\n", i.queryName())); + throw; + } + } + + printMissing(*globals.state, newElems); + + if (globals.dryRun) return; + + if (createUserEnv(*globals.state, newElems, + globals.profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opUpgrade(Globals & globals, Strings opFlags, Strings opArgs) +{ + UpgradeType upgradeType = utLt; + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; + else if (arg == "--lt") upgradeType = utLt; + else if (arg == "--leq") upgradeType = utLeq; + else if (arg == "--eq") upgradeType = utEq; + else if (arg == "--always") upgradeType = utAlways; + else throw UsageError(format("unknown flag '%1%'") % arg); + } + + upgradeDerivations(globals, opArgs, upgradeType); +} + + +static void setMetaFlag(EvalState & state, DrvInfo & drv, + const string & name, const string & value) +{ + Value * v = state.allocValue(); + mkString(*v, value.c_str()); + drv.setMeta(name, v); +} + + +static void opSetFlag(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() < 2) + throw UsageError("not enough arguments to '--set-flag'"); + + Strings::iterator arg = opArgs.begin(); + string flagName = *arg++; + string flagValue = *arg++; + DrvNames selectors = drvNamesFromArgs(Strings(arg, opArgs.end())); + + while (true) { + string lockToken = optimisticLockProfile(globals.profile); + + DrvInfos installedElems = queryInstalled(*globals.state, globals.profile); + + /* Update all matching derivations. */ + for (auto & i : installedElems) { + DrvName drvName(i.queryName()); + for (auto & j : selectors) + if (j.matches(drvName)) { + printInfo("setting flag on '%1%'", i.queryName()); + j.hits++; + setMetaFlag(*globals.state, i, flagName, flagValue); + break; + } + } + + checkSelectorUse(selectors); + + /* Write the new user environment. */ + if (createUserEnv(*globals.state, installedElems, + globals.profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opSet(Globals & globals, Strings opFlags, Strings opArgs) +{ + auto store2 = globals.state->store.dynamic_pointer_cast(); + if (!store2) throw Error("--set is not supported for this Nix store"); + + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (parseInstallSourceOptions(globals, i, opFlags, arg)) ; + else throw UsageError(format("unknown flag '%1%'") % arg); + } + + DrvInfos elems; + queryInstSources(*globals.state, globals.instSource, opArgs, elems, true); + + if (elems.size() != 1) + throw Error("--set requires exactly one derivation"); + + DrvInfo & drv(elems.front()); + + if (globals.forceName != "") + drv.setName(globals.forceName); + + if (drv.queryDrvPath() != "") { + PathSet paths = {drv.queryDrvPath()}; + printMissing(globals.state->store, paths); + if (globals.dryRun) return; + globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal); + } + else { + printMissing(globals.state->store, {drv.queryOutPath()}); + if (globals.dryRun) return; + globals.state->store->ensurePath(drv.queryOutPath()); + } + + debug(format("switching to new user environment")); + Path generation = createGeneration(ref(store2), globals.profile, drv.queryOutPath()); + switchLink(globals.profile, generation); +} + + +static void uninstallDerivations(Globals & globals, Strings & selectors, + Path & profile) +{ + while (true) { + string lockToken = optimisticLockProfile(profile); + + DrvInfos installedElems = queryInstalled(*globals.state, profile); + DrvInfos newElems; + + for (auto & i : installedElems) { + DrvName drvName(i.queryName()); + bool found = false; + for (auto & j : selectors) + /* !!! the repeated calls to followLinksToStorePath() + are expensive, should pre-compute them. */ + if ((isPath(j) && i.queryOutPath() == globals.state->store->followLinksToStorePath(j)) + || DrvName(j).matches(drvName)) + { + printInfo("uninstalling '%s'", i.queryName()); + found = true; + break; + } + if (!found) newElems.push_back(i); + } + + if (globals.dryRun) return; + + if (createUserEnv(*globals.state, newElems, + profile, settings.envKeepDerivations, lockToken)) break; + } +} + + +static void opUninstall(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + uninstallDerivations(globals, opArgs, globals.profile); +} + + +static bool cmpChars(char a, char b) +{ + return toupper(a) < toupper(b); +} + + +static bool cmpElemByName(const DrvInfo & a, const DrvInfo & b) +{ + auto a_name = a.queryName(); + auto b_name = b.queryName(); + return lexicographical_compare( + a_name.begin(), a_name.end(), + b_name.begin(), b_name.end(), cmpChars); +} + + +typedef list Table; + + +void printTable(Table & table) +{ + auto nrColumns = table.size() > 0 ? table.front().size() : 0; + + vector widths; + widths.resize(nrColumns); + + for (auto & i : table) { + assert(i.size() == nrColumns); + Strings::iterator j; + size_t column; + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) + if (j->size() > widths[column]) widths[column] = j->size(); + } + + for (auto & i : table) { + Strings::iterator j; + size_t column; + for (j = i.begin(), column = 0; j != i.end(); ++j, ++column) { + string s = *j; + replace(s.begin(), s.end(), '\n', ' '); + cout << s; + if (column < nrColumns - 1) + cout << string(widths[column] - s.size() + 2, ' '); + } + cout << std::endl; + } +} + + +/* This function compares the version of an element against the + versions in the given set of elements. `cvLess' means that only + lower versions are in the set, `cvEqual' means that at most an + equal version is in the set, and `cvGreater' means that there is at + least one element with a higher version in the set. `cvUnavail' + means that there are no elements with the same name in the set. */ + +typedef enum { cvLess, cvEqual, cvGreater, cvUnavail } VersionDiff; + +static VersionDiff compareVersionAgainstSet( + const DrvInfo & elem, const DrvInfos & elems, string & version) +{ + DrvName name(elem.queryName()); + + VersionDiff diff = cvUnavail; + version = "?"; + + for (auto & i : elems) { + DrvName name2(i.queryName()); + if (name.name == name2.name) { + int d = compareVersions(name.version, name2.version); + if (d < 0) { + diff = cvGreater; + version = name2.version; + } + else if (diff != cvGreater && d == 0) { + diff = cvEqual; + version = name2.version; + } + else if (diff != cvGreater && diff != cvEqual && d > 0) { + diff = cvLess; + if (version == "" || compareVersions(version, name2.version) < 0) + version = name2.version; + } + } + } + + return diff; +} + + +static void queryJSON(Globals & globals, vector & elems) +{ + JSONObject topObj(cout, true); + for (auto & i : elems) { + JSONObject pkgObj = topObj.object(i.attrPath); + + auto drvName = DrvName(i.queryName()); + pkgObj.attr("name", drvName.fullName); + pkgObj.attr("pname", drvName.name); + pkgObj.attr("version", drvName.version); + pkgObj.attr("system", i.querySystem()); + + JSONObject metaObj = pkgObj.object("meta"); + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + auto placeholder = metaObj.placeholder(j); + Value * v = i.queryMeta(j); + if (!v) { + printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); + placeholder.write(nullptr); + } else { + PathSet context; + printValueAsJSON(*globals.state, true, *v, placeholder, context); + } + } + } +} + + +static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) +{ + Strings remaining; + string attrPath; + + bool printStatus = false; + bool printName = true; + bool printAttrPath = false; + bool printSystem = false; + bool printDrvPath = false; + bool printOutPath = false; + bool printDescription = false; + bool printMeta = false; + bool compareVersions = false; + bool xmlOutput = false; + bool jsonOutput = false; + + enum { sInstalled, sAvailable } source = sInstalled; + + settings.readOnlyMode = true; /* makes evaluation a bit faster */ + + for (Strings::iterator i = opFlags.begin(); i != opFlags.end(); ) { + string arg = *i++; + if (arg == "--status" || arg == "-s") printStatus = true; + else if (arg == "--no-name") printName = false; + else if (arg == "--system") printSystem = true; + else if (arg == "--description") printDescription = true; + else if (arg == "--compare-versions" || arg == "-c") compareVersions = true; + else if (arg == "--drv-path") printDrvPath = true; + else if (arg == "--out-path") printOutPath = true; + else if (arg == "--meta") printMeta = true; + else if (arg == "--installed") source = sInstalled; + else if (arg == "--available" || arg == "-a") source = sAvailable; + else if (arg == "--xml") xmlOutput = true; + else if (arg == "--json") jsonOutput = true; + else if (arg == "--attr-path" || arg == "-P") printAttrPath = true; + else if (arg == "--attr" || arg == "-A") + attrPath = needArg(i, opFlags, arg); + else + throw UsageError(format("unknown flag '%1%'") % arg); + } + + + /* Obtain derivation information from the specified source. */ + DrvInfos availElems, installedElems; + + if (source == sInstalled || compareVersions || printStatus) + installedElems = queryInstalled(*globals.state, globals.profile); + + if (source == sAvailable || compareVersions) + loadDerivations(*globals.state, globals.instSource.nixExprPath, + globals.instSource.systemFilter, *globals.instSource.autoArgs, + attrPath, availElems); + + DrvInfos elems_ = filterBySelector(*globals.state, + source == sInstalled ? installedElems : availElems, + opArgs, false); + + DrvInfos & otherElems(source == sInstalled ? availElems : installedElems); + + + /* Sort them by name. */ + /* !!! */ + vector elems; + for (auto & i : elems_) elems.push_back(i); + sort(elems.begin(), elems.end(), cmpElemByName); + + + /* We only need to know the installed paths when we are querying + the status of the derivation. */ + PathSet installed; /* installed paths */ + + if (printStatus) { + for (auto & i : installedElems) + installed.insert(i.queryOutPath()); + } + + + /* Query which paths have substitutes. */ + PathSet validPaths, substitutablePaths; + if (printStatus || globals.prebuiltOnly) { + PathSet paths; + for (auto & i : elems) + try { + paths.insert(i.queryOutPath()); + } catch (AssertionError & e) { + printMsg(lvlTalkative, "skipping derivation named '%s' which gives an assertion failure", i.queryName()); + i.setFailed(); + } + validPaths = globals.state->store->queryValidPaths(paths); + substitutablePaths = globals.state->store->querySubstitutablePaths(paths); + } + + + /* Print the desired columns, or XML output. */ + if (jsonOutput) { + queryJSON(globals, elems); + return; + } + + bool tty = isatty(STDOUT_FILENO); + RunPager pager; + + Table table; + std::ostringstream dummy; + XMLWriter xml(true, *(xmlOutput ? &cout : &dummy)); + XMLOpenElement xmlRoot(xml, "items"); + + for (auto & i : elems) { + try { + if (i.hasFailed()) continue; + + //Activity act(*logger, lvlDebug, format("outputting query result '%1%'") % i.attrPath); + + if (globals.prebuiltOnly && + validPaths.find(i.queryOutPath()) == validPaths.end() && + substitutablePaths.find(i.queryOutPath()) == substitutablePaths.end()) + continue; + + /* For table output. */ + Strings columns; + + /* For XML output. */ + XMLAttrs attrs; + + if (printStatus) { + Path outPath = i.queryOutPath(); + bool hasSubs = substitutablePaths.find(outPath) != substitutablePaths.end(); + bool isInstalled = installed.find(outPath) != installed.end(); + bool isValid = validPaths.find(outPath) != validPaths.end(); + if (xmlOutput) { + attrs["installed"] = isInstalled ? "1" : "0"; + attrs["valid"] = isValid ? "1" : "0"; + attrs["substitutable"] = hasSubs ? "1" : "0"; + } else + columns.push_back( + (string) (isInstalled ? "I" : "-") + + (isValid ? "P" : "-") + + (hasSubs ? "S" : "-")); + } + + if (xmlOutput) + attrs["attrPath"] = i.attrPath; + else if (printAttrPath) + columns.push_back(i.attrPath); + + if (xmlOutput) { + auto drvName = DrvName(i.queryName()); + attrs["name"] = drvName.fullName; + attrs["pname"] = drvName.name; + attrs["version"] = drvName.version; + } else if (printName) { + columns.push_back(i.queryName()); + } + + if (compareVersions) { + /* Compare this element against the versions of the + same named packages in either the set of available + elements, or the set of installed elements. !!! + This is O(N * M), should be O(N * lg M). */ + string version; + VersionDiff diff = compareVersionAgainstSet(i, otherElems, version); + + char ch; + switch (diff) { + case cvLess: ch = '>'; break; + case cvEqual: ch = '='; break; + case cvGreater: ch = '<'; break; + case cvUnavail: ch = '-'; break; + default: abort(); + } + + if (xmlOutput) { + if (diff != cvUnavail) { + attrs["versionDiff"] = ch; + attrs["maxComparedVersion"] = version; + } + } else { + string column = (string) "" + ch + " " + version; + if (diff == cvGreater && tty) + column = ANSI_RED + column + ANSI_NORMAL; + columns.push_back(column); + } + } + + if (xmlOutput) { + if (i.querySystem() != "") attrs["system"] = i.querySystem(); + } + else if (printSystem) + columns.push_back(i.querySystem()); + + if (printDrvPath) { + string drvPath = i.queryDrvPath(); + if (xmlOutput) { + if (drvPath != "") attrs["drvPath"] = drvPath; + } else + columns.push_back(drvPath == "" ? "-" : drvPath); + } + + if (printOutPath && !xmlOutput) { + DrvInfo::Outputs outputs = i.queryOutputs(); + string s; + for (auto & j : outputs) { + if (!s.empty()) s += ';'; + if (j.first != "out") { s += j.first; s += "="; } + s += j.second; + } + columns.push_back(s); + } + + if (printDescription) { + string descr = i.queryMetaString("description"); + if (xmlOutput) { + if (descr != "") attrs["description"] = descr; + } else + columns.push_back(descr); + } + + if (xmlOutput) { + if (printOutPath || printMeta) { + XMLOpenElement item(xml, "item", attrs); + if (printOutPath) { + DrvInfo::Outputs outputs = i.queryOutputs(); + for (auto & j : outputs) { + XMLAttrs attrs2; + attrs2["name"] = j.first; + attrs2["path"] = j.second; + xml.writeEmptyElement("output", attrs2); + } + } + if (printMeta) { + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + XMLAttrs attrs2; + attrs2["name"] = j; + Value * v = i.queryMeta(j); + if (!v) + printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j); + else { + if (v->type == tString) { + attrs2["type"] = "string"; + attrs2["value"] = v->string.s; + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tInt) { + attrs2["type"] = "int"; + attrs2["value"] = (format("%1%") % v->integer).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tFloat) { + attrs2["type"] = "float"; + attrs2["value"] = (format("%1%") % v->fpoint).str(); + xml.writeEmptyElement("meta", attrs2); + } else if (v->type == tBool) { + attrs2["type"] = "bool"; + attrs2["value"] = v->boolean ? "true" : "false"; + xml.writeEmptyElement("meta", attrs2); + } else if (v->isList()) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + for (unsigned int j = 0; j < v->listSize(); ++j) { + if (v->listElems()[j]->type != tString) continue; + XMLAttrs attrs3; + attrs3["value"] = v->listElems()[j]->string.s; + xml.writeEmptyElement("string", attrs3); + } + } else if (v->type == tAttrs) { + attrs2["type"] = "strings"; + XMLOpenElement m(xml, "meta", attrs2); + Bindings & attrs = *v->attrs; + for (auto &i : attrs) { + Attr & a(*attrs.find(i.name)); + if(a.value->type != tString) continue; + XMLAttrs attrs3; + attrs3["type"] = i.name; + attrs3["value"] = a.value->string.s; + xml.writeEmptyElement("string", attrs3); + } + } + } + } + } + } else + xml.writeEmptyElement("item", attrs); + } else + table.push_back(columns); + + cout.flush(); + + } catch (AssertionError & e) { + printMsg(lvlTalkative, "skipping derivation named '%1%' which gives an assertion failure", i.queryName()); + } catch (Error & e) { + e.addPrefix(fmt("while querying the derivation named '%1%':\n", i.queryName())); + throw; + } + } + + if (!xmlOutput) printTable(table); +} + + +static void opSwitchProfile(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 1) + throw UsageError(format("exactly one argument expected")); + + Path profile = absPath(opArgs.front()); + Path profileLink = getHome() + "/.nix-profile"; + + switchLink(profileLink, profile); +} + + +static const int prevGen = -2; + + +static void switchGeneration(Globals & globals, int dstGen) +{ + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + Generation dst; + for (auto & i : gens) + if ((dstGen == prevGen && i.number < curGen) || + (dstGen >= 0 && i.number == dstGen)) + dst = i; + + if (!dst) { + if (dstGen == prevGen) + throw Error(format("no generation older than the current (%1%) exists") + % curGen); + else + throw Error(format("generation %1% does not exist") % dstGen); + } + + printInfo(format("switching from generation %1% to %2%") + % curGen % dst.number); + + if (globals.dryRun) return; + + switchLink(globals.profile, dst.path); +} + + +static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 1) + throw UsageError(format("exactly one argument expected")); + + int dstGen; + if (!string2Int(opArgs.front(), dstGen)) + throw UsageError(format("expected a generation number")); + + switchGeneration(globals, dstGen); +} + + +static void opRollback(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 0) + throw UsageError(format("no arguments expected")); + + switchGeneration(globals, prevGen); +} + + +static void opListGenerations(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + if (opArgs.size() != 0) + throw UsageError(format("no arguments expected")); + + PathLocks lock; + lockProfile(lock, globals.profile); + + int curGen; + Generations gens = findGenerations(globals.profile, curGen); + + RunPager pager; + + for (auto & i : gens) { + tm t; + if (!localtime_r(&i.creationTime, &t)) throw Error("cannot convert time"); + cout << format("%|4| %|4|-%|02|-%|02| %|02|:%|02|:%|02| %||\n") + % i.number + % (t.tm_year + 1900) % (t.tm_mon + 1) % t.tm_mday + % t.tm_hour % t.tm_min % t.tm_sec + % (i.number == curGen ? "(current)" : ""); + } +} + + +static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opArgs) +{ + if (opFlags.size() > 0) + throw UsageError(format("unknown flag '%1%'") % opFlags.front()); + + if (opArgs.size() == 1 && opArgs.front() == "old") { + deleteOldGenerations(globals.profile, globals.dryRun); + } else if (opArgs.size() == 1 && opArgs.front().find('d') != string::npos) { + deleteGenerationsOlderThan(globals.profile, opArgs.front(), globals.dryRun); + } else if (opArgs.size() == 1 && opArgs.front().find('+') != string::npos) { + if(opArgs.front().size() < 2) + throw Error(format("invalid number of generations ‘%1%’") % opArgs.front()); + string str_max = string(opArgs.front(), 1, opArgs.front().size()); + int max; + if (!string2Int(str_max, max) || max == 0) + throw Error(format("invalid number of generations to keep ‘%1%’") % opArgs.front()); + deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + } else { + std::set gens; + for (auto & i : opArgs) { + unsigned int n; + if (!string2Int(i, n)) + throw UsageError(format("invalid generation number '%1%'") % i); + gens.insert(n); + } + deleteGenerations(globals.profile, gens, globals.dryRun); + } +} + + +static void opVersion(Globals & globals, Strings opFlags, Strings opArgs) +{ + printVersion("nix-env"); +} + + +static int _main(int argc, char * * argv) +{ + { + Strings opFlags, opArgs; + Operation op = 0; + RepairFlag repair = NoRepair; + string file; + + Globals globals; + + globals.instSource.type = srcUnknown; + globals.instSource.nixExprPath = getHome() + "/.nix-defexpr"; + globals.instSource.systemFilter = "*"; + + if (!pathExists(globals.instSource.nixExprPath)) { + try { + createDirs(globals.instSource.nixExprPath); + replaceSymlink( + fmt("%s/profiles/per-user/%s/channels", settings.nixStateDir, getUserName()), + globals.instSource.nixExprPath + "/channels"); + if (getuid() != 0) + replaceSymlink( + fmt("%s/profiles/per-user/root/channels", settings.nixStateDir), + globals.instSource.nixExprPath + "/channels_root"); + } catch (Error &) { } + } + + globals.dryRun = false; + globals.preserveInstalled = false; + globals.removeAll = false; + globals.prebuiltOnly = false; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + Operation oldOp = op; + + if (*arg == "--help") + showManPage("nix-env"); + else if (*arg == "--version") + op = opVersion; + else if (*arg == "--install" || *arg == "-i") + op = opInstall; + else if (*arg == "--force-name") // undocumented flag for nix-install-package + globals.forceName = getArg(*arg, arg, end); + else if (*arg == "--uninstall" || *arg == "-e") + op = opUninstall; + else if (*arg == "--upgrade" || *arg == "-u") + op = opUpgrade; + else if (*arg == "--set-flag") + op = opSetFlag; + else if (*arg == "--set") + op = opSet; + else if (*arg == "--query" || *arg == "-q") + op = opQuery; + else if (*arg == "--profile" || *arg == "-p") + globals.profile = absPath(getArg(*arg, arg, end)); + else if (*arg == "--file" || *arg == "-f") + file = getArg(*arg, arg, end); + else if (*arg == "--switch-profile" || *arg == "-S") + op = opSwitchProfile; + else if (*arg == "--switch-generation" || *arg == "-G") + op = opSwitchGeneration; + else if (*arg == "--rollback") + op = opRollback; + else if (*arg == "--list-generations") + op = opListGenerations; + else if (*arg == "--delete-generations") + op = opDeleteGenerations; + else if (*arg == "--dry-run") { + printInfo("(dry run; not doing anything)"); + globals.dryRun = true; + } + else if (*arg == "--system-filter") + globals.instSource.systemFilter = getArg(*arg, arg, end); + else if (*arg == "--prebuilt-only" || *arg == "-b") + globals.prebuiltOnly = true; + else if (*arg == "--repair") + repair = Repair; + else if (*arg != "" && arg->at(0) == '-') { + opFlags.push_back(*arg); + /* FIXME: hacky */ + if (*arg == "--from-profile" || + (op == opQuery && (*arg == "--attr" || *arg == "-A"))) + opFlags.push_back(getArg(*arg, arg, end)); + } + else + opArgs.push_back(*arg); + + if (oldOp && oldOp != op) + throw UsageError("only one operation may be specified"); + + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (!op) throw UsageError("no operation specified"); + + auto store = openStore(); + + globals.state = std::shared_ptr(new EvalState(myArgs.searchPath, store)); + globals.state->repair = repair; + + if (file != "") + globals.instSource.nixExprPath = lookupFileArg(*globals.state, file); + + globals.instSource.autoArgs = myArgs.getAutoArgs(*globals.state); + + if (globals.profile == "") + globals.profile = getEnv("NIX_PROFILE", ""); + + if (globals.profile == "") { + Path profileLink = getHome() + "/.nix-profile"; + try { + if (!pathExists(profileLink)) { + replaceSymlink( + getuid() == 0 + ? settings.nixStateDir + "/profiles/default" + : fmt("%s/profiles/per-user/%s/profile", settings.nixStateDir, getUserName()), + profileLink); + } + globals.profile = absPath(readLink(profileLink), dirOf(profileLink)); + } catch (Error &) { + globals.profile = profileLink; + } + } + + op(globals, opFlags, opArgs); + + globals.state->printStats(); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-env", _main); diff --git a/third_party/nix/src/nix-env/user-env.cc b/third_party/nix/src/nix-env/user-env.cc new file mode 100644 index 000000000..7b9a88281 --- /dev/null +++ b/third_party/nix/src/nix-env/user-env.cc @@ -0,0 +1,156 @@ +#include "user-env.hh" +#include "util.hh" +#include "derivations.hh" +#include "store-api.hh" +#include "globals.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "profiles.hh" + + +namespace nix { + + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv) +{ + DrvInfos elems; + Path manifestFile = userEnv + "/manifest.nix"; + if (pathExists(manifestFile)) { + Value v; + state.evalFile(manifestFile, v); + Bindings & bindings(*state.allocBindings(0)); + getDerivations(state, v, "", bindings, elems, false); + } + return elems; +} + + +bool createUserEnv(EvalState & state, DrvInfos & elems, + const Path & profile, bool keepDerivations, + const string & lockToken) +{ + /* Build the components in the user environment, if they don't + exist already. */ + PathSet drvsToBuild; + for (auto & i : elems) + if (i.queryDrvPath() != "") + drvsToBuild.insert(i.queryDrvPath()); + + debug(format("building user environment dependencies")); + state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal); + + /* Construct the whole top level derivation. */ + PathSet references; + Value manifest; + state.mkList(manifest, elems.size()); + unsigned int n = 0; + for (auto & i : elems) { + /* Create a pseudo-derivation containing the name, system, + output paths, and optionally the derivation path, as well + as the meta attributes. */ + Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + + Value & v(*state.allocValue()); + manifest.listElems()[n++] = &v; + state.mkAttrs(v, 16); + + mkString(*state.allocAttr(v, state.sType), "derivation"); + mkString(*state.allocAttr(v, state.sName), i.queryName()); + auto system = i.querySystem(); + if (!system.empty()) + mkString(*state.allocAttr(v, state.sSystem), system); + mkString(*state.allocAttr(v, state.sOutPath), i.queryOutPath()); + if (drvPath != "") + mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); + + // Copy each output meant for installation. + DrvInfo::Outputs outputs = i.queryOutputs(true); + Value & vOutputs = *state.allocAttr(v, state.sOutputs); + state.mkList(vOutputs, outputs.size()); + unsigned int m = 0; + for (auto & j : outputs) { + mkString(*(vOutputs.listElems()[m++] = state.allocValue()), j.first); + Value & vOutputs = *state.allocAttr(v, state.symbols.create(j.first)); + state.mkAttrs(vOutputs, 2); + mkString(*state.allocAttr(vOutputs, state.sOutPath), j.second); + + /* This is only necessary when installing store paths, e.g., + `nix-env -i /nix/store/abcd...-foo'. */ + state.store->addTempRoot(j.second); + state.store->ensurePath(j.second); + + references.insert(j.second); + } + + // Copy the meta attributes. + Value & vMeta = *state.allocAttr(v, state.sMeta); + state.mkAttrs(vMeta, 16); + StringSet metaNames = i.queryMetaNames(); + for (auto & j : metaNames) { + Value * v = i.queryMeta(j); + if (!v) continue; + vMeta.attrs->push_back(Attr(state.symbols.create(j), v)); + } + vMeta.attrs->sort(); + v.attrs->sort(); + + if (drvPath != "") references.insert(drvPath); + } + + /* Also write a copy of the list of user environment elements to + the store; we need it for future modifications of the + environment. */ + Path manifestFile = state.store->addTextToStore("env-manifest.nix", + (format("%1%") % manifest).str(), references); + + /* Get the environment builder expression. */ + Value envBuilder; + state.evalFile(state.findFile("nix/buildenv.nix"), envBuilder); + + /* Construct a Nix expression that calls the user environment + builder with the manifest as argument. */ + Value args, topLevel; + state.mkAttrs(args, 3); + mkString(*state.allocAttr(args, state.symbols.create("manifest")), + manifestFile, {manifestFile}); + args.attrs->push_back(Attr(state.symbols.create("derivations"), &manifest)); + args.attrs->sort(); + mkApp(topLevel, envBuilder, args); + + /* Evaluate it. */ + debug("evaluating user environment builder"); + state.forceValue(topLevel); + PathSet context; + Attr & aDrvPath(*topLevel.attrs->find(state.sDrvPath)); + Path topLevelDrv = state.coerceToPath(aDrvPath.pos ? *(aDrvPath.pos) : noPos, *(aDrvPath.value), context); + Attr & aOutPath(*topLevel.attrs->find(state.sOutPath)); + Path topLevelOut = state.coerceToPath(aOutPath.pos ? *(aOutPath.pos) : noPos, *(aOutPath.value), context); + + /* Realise the resulting store expression. */ + debug("building user environment"); + state.store->buildPaths({topLevelDrv}, state.repair ? bmRepair : bmNormal); + + /* Switch the current user environment to the output path. */ + auto store2 = state.store.dynamic_pointer_cast(); + + if (store2) { + PathLocks lock; + lockProfile(lock, profile); + + Path lockTokenCur = optimisticLockProfile(profile); + if (lockToken != lockTokenCur) { + printError(format("profile '%1%' changed while we were busy; restarting") % profile); + return false; + } + + debug(format("switching to new user environment")); + Path generation = createGeneration(ref(store2), profile, topLevelOut); + switchLink(profile, generation); + } + + return true; +} + + +} diff --git a/third_party/nix/src/nix-env/user-env.hh b/third_party/nix/src/nix-env/user-env.hh new file mode 100644 index 000000000..f188efe9b --- /dev/null +++ b/third_party/nix/src/nix-env/user-env.hh @@ -0,0 +1,13 @@ +#pragma once + +#include "get-drvs.hh" + +namespace nix { + +DrvInfos queryInstalled(EvalState & state, const Path & userEnv); + +bool createUserEnv(EvalState & state, DrvInfos & elems, + const Path & profile, bool keepDerivations, + const string & lockToken); + +} diff --git a/third_party/nix/src/nix-instantiate/nix-instantiate.cc b/third_party/nix/src/nix-instantiate/nix-instantiate.cc new file mode 100644 index 000000000..a736caa8f --- /dev/null +++ b/third_party/nix/src/nix-instantiate/nix-instantiate.cc @@ -0,0 +1,196 @@ +#include "globals.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "get-drvs.hh" +#include "attr-path.hh" +#include "value-to-xml.hh" +#include "value-to-json.hh" +#include "util.hh" +#include "store-api.hh" +#include "common-eval-args.hh" +#include "legacy.hh" + +#include +#include + + +using namespace nix; + + +static Path gcRoot; +static int rootNr = 0; +static bool indirectRoot = false; + + +enum OutputKind { okPlain, okXML, okJSON }; + + +void processExpr(EvalState & state, const Strings & attrPaths, + bool parseOnly, bool strict, Bindings & autoArgs, + bool evalOnly, OutputKind output, bool location, Expr * e) +{ + if (parseOnly) { + std::cout << format("%1%\n") % *e; + return; + } + + Value vRoot; + state.eval(e, vRoot); + + for (auto & i : attrPaths) { + Value & v(*findAlongAttrPath(state, i, autoArgs, vRoot)); + state.forceValue(v); + + PathSet context; + if (evalOnly) { + Value vRes; + if (autoArgs.empty()) + vRes = v; + else + state.autoCallFunction(autoArgs, v, vRes); + if (output == okXML) + printValueAsXML(state, strict, location, vRes, std::cout, context); + else if (output == okJSON) + printValueAsJSON(state, strict, vRes, std::cout, context); + else { + if (strict) state.forceValueDeep(vRes); + std::cout << vRes << std::endl; + } + } else { + DrvInfos drvs; + getDerivations(state, v, "", autoArgs, drvs, false); + for (auto & i : drvs) { + Path drvPath = i.queryDrvPath(); + + /* What output do we want? */ + string outputName = i.queryOutputName(); + if (outputName == "") + throw Error(format("derivation '%1%' lacks an 'outputName' attribute ") % drvPath); + + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = indirectRoot ? absPath(gcRoot) : gcRoot; + if (++rootNr > 1) rootName += "-" + std::to_string(rootNr); + auto store2 = state.store.dynamic_pointer_cast(); + if (store2) + drvPath = store2->addPermRoot(drvPath, rootName, indirectRoot); + } + std::cout << format("%1%%2%\n") % drvPath % (outputName != "out" ? "!" + outputName : ""); + } + } + } +} + + +static int _main(int argc, char * * argv) +{ + { + Strings files; + bool readStdin = false; + bool fromArgs = false; + bool findFile = false; + bool evalOnly = false; + bool parseOnly = false; + OutputKind outputKind = okPlain; + bool xmlOutputSourceLocation = true; + bool strict = false; + Strings attrPaths; + bool wantsReadWrite = false; + RepairFlag repair = NoRepair; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-instantiate"); + else if (*arg == "--version") + printVersion("nix-instantiate"); + else if (*arg == "-") + readStdin = true; + else if (*arg == "--expr" || *arg == "-E") + fromArgs = true; + else if (*arg == "--eval" || *arg == "--eval-only") + evalOnly = true; + else if (*arg == "--read-write-mode") + wantsReadWrite = true; + else if (*arg == "--parse" || *arg == "--parse-only") + parseOnly = evalOnly = true; + else if (*arg == "--find-file") + findFile = true; + else if (*arg == "--attr" || *arg == "-A") + attrPaths.push_back(getArg(*arg, arg, end)); + else if (*arg == "--add-root") + gcRoot = getArg(*arg, arg, end); + else if (*arg == "--indirect") + indirectRoot = true; + else if (*arg == "--xml") + outputKind = okXML; + else if (*arg == "--json") + outputKind = okJSON; + else if (*arg == "--no-location") + xmlOutputSourceLocation = false; + else if (*arg == "--strict") + strict = true; + else if (*arg == "--repair") + repair = Repair; + else if (*arg == "--dry-run") + settings.readOnlyMode = true; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + files.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (evalOnly && !wantsReadWrite) + settings.readOnlyMode = true; + + auto store = openStore(); + + auto state = std::make_unique(myArgs.searchPath, store); + state->repair = repair; + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + if (attrPaths.empty()) attrPaths = {""}; + + if (findFile) { + for (auto & i : files) { + Path p = state->findFile(i); + if (p == "") throw Error(format("unable to find '%1%'") % i); + std::cout << p << std::endl; + } + return 0; + } + + if (readStdin) { + Expr * e = state->parseStdin(); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, + evalOnly, outputKind, xmlOutputSourceLocation, e); + } else if (files.empty() && !fromArgs) + files.push_back("./default.nix"); + + for (auto & i : files) { + Expr * e = fromArgs + ? state->parseExprFromString(i, absPath(".")) + : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + processExpr(*state, attrPaths, parseOnly, strict, autoArgs, + evalOnly, outputKind, xmlOutputSourceLocation, e); + } + + state->printStats(); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-instantiate", _main); diff --git a/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc b/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc new file mode 100644 index 000000000..f54706a8a --- /dev/null +++ b/third_party/nix/src/nix-prefetch-url/nix-prefetch-url.cc @@ -0,0 +1,236 @@ +#include "hash.hh" +#include "shared.hh" +#include "download.hh" +#include "store-api.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "common-eval-args.hh" +#include "attr-path.hh" +#include "legacy.hh" +#include "finally.hh" +#include "progress-bar.hh" + +#include + +#include +#include +#include + +using namespace nix; + + +/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUri(EvalState & state, string uri) +{ + if (string(uri, 0, 9) != "mirror://") return uri; + + string s(uri, 9); + auto p = s.find('/'); + if (p == string::npos) throw Error("invalid mirror URI"); + string mirrorName(s, 0, p); + + Value vMirrors; + state.eval(state.parseExprFromString("import ", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error(format("unknown mirror name '%1%'") % mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error(format("mirror URI '%1%' did not expand to anything") % uri); + + string mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + + +static int _main(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector args; + bool printPath = getEnv("PRINT_PATH") != ""; + bool fromExpr = false; + string attrPath; + bool unpack = false; + string name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(baseNameOf(argv[0]), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type '%1%'") % s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URI from the specified Nix + expression. */ + string uri; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URI"); + uri = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot)); + state->forceAttrs(v); + + /* Extract the URI. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + uri = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (name.empty()) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + /* Figure out a name in the Nix store. */ + if (name.empty()) + name = baseNameOf(uri); + if (name.empty()) + throw Error(format("cannot figure out file name for '%1%'") % uri); + + /* If an expected hash is given, the file may already exist in + the store. */ + Hash hash, expectedHash(ht); + Path storePath; + if (args.size() == 2) { + expectedHash = Hash(args[1], ht); + storePath = store->makeFixedOutputPath(unpack, expectedHash, name); + if (store->isValidPath(storePath)) + hash = expectedHash; + else + storePath.clear(); + } + + if (storePath.empty()) { + + auto actualUri = resolveMirrorUri(*state, uri); + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, 0600); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + DownloadRequest req(actualUri); + req.decompress = false; + getDownloader()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + printInfo("unpacking..."); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + if (hasSuffix(baseNameOf(uri), ".zip")) + runProgram("unzip", true, {"-qq", tmpFile, "-d", unpacked}); + else + // FIXME: this requires GNU tar for decompression. + runProgram("tar", true, {"xf", tmpFile, "-C", unpacked}); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + /* FIXME: inefficient; addToStore() will also hash + this. */ + hash = unpack ? hashPath(ht, tmpFile).first : hashFile(ht, tmpFile); + + if (expectedHash != Hash(ht) && expectedHash != hash) + throw Error(format("hash mismatch for '%1%'") % uri); + + /* Copy the file to the Nix store. FIXME: if RemoteStore + implemented addToStoreFromDump() and downloadFile() + supported a sink, we could stream the download directly + into the Nix store. */ + storePath = store->addToStore(name, tmpFile, unpack, ht); + + assert(storePath == store->makeFixedOutputPath(unpack, hash, name)); + } + + stopProgressBar(); + + if (!printPath) + printInfo(format("path is '%1%'") % storePath); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << storePath << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-prefetch-url", _main); diff --git a/third_party/nix/src/nix-store/dotgraph.cc b/third_party/nix/src/nix-store/dotgraph.cc new file mode 100644 index 000000000..abdfa5e58 --- /dev/null +++ b/third_party/nix/src/nix-store/dotgraph.cc @@ -0,0 +1,155 @@ +#include "dotgraph.hh" +#include "util.hh" +#include "store-api.hh" + +#include + + +using std::cout; + +namespace nix { + + +static string dotQuote(const string & s) +{ + return "\"" + s + "\""; +} + + +static string nextColour() +{ + static int n = 0; + static string colours[] = + { "black", "red", "green", "blue" + , "magenta", "burlywood" }; + return colours[n++ % (sizeof(colours) / sizeof(string))]; +} + + +static string makeEdge(const string & src, const string & dst) +{ + format f = format("%1% -> %2% [color = %3%];\n") + % dotQuote(src) % dotQuote(dst) % dotQuote(nextColour()); + return f.str(); +} + + +static string makeNode(const string & id, const string & label, + const string & colour) +{ + format f = format("%1% [label = %2%, shape = box, " + "style = filled, fillcolor = %3%];\n") + % dotQuote(id) % dotQuote(label) % dotQuote(colour); + return f.str(); +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + return string(p, p.find('-') + 1); +} + + +#if 0 +string pathLabel(const Path & nePath, const string & elemPath) +{ + return (string) nePath + "-" + elemPath; +} + + +void printClosure(const Path & nePath, const StoreExpr & fs) +{ + PathSet workList(fs.closure.roots); + PathSet doneSet; + + for (PathSet::iterator i = workList.begin(); i != workList.end(); ++i) { + cout << makeEdge(pathLabel(nePath, *i), nePath); + } + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) == doneSet.end()) { + doneSet.insert(path); + + ClosureElems::const_iterator elem = fs.closure.elems.find(path); + if (elem == fs.closure.elems.end()) + throw Error(format("bad closure, missing path '%1%'") % path); + + for (StringSet::const_iterator i = elem->second.refs.begin(); + i != elem->second.refs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(pathLabel(nePath, *i), pathLabel(nePath, path)); + } + + cout << makeNode(pathLabel(nePath, path), + symbolicName(path), "#ff0000"); + } + } +} +#endif + + +void printDotGraph(ref store, const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + + cout << "digraph G {\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + if (doneSet.find(path) != doneSet.end()) continue; + doneSet.insert(path); + + cout << makeNode(path, symbolicName(path), "#ff0000"); + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(p, path); + } + } + +#if 0 + StoreExpr ne = storeExprFromPath(path); + + string label, colour; + + if (ne.type == StoreExpr::neDerivation) { + for (PathSet::iterator i = ne.derivation.inputs.begin(); + i != ne.derivation.inputs.end(); ++i) + { + workList.insert(*i); + cout << makeEdge(*i, path); + } + + label = "derivation"; + colour = "#00ff00"; + for (StringPairs::iterator i = ne.derivation.env.begin(); + i != ne.derivation.env.end(); ++i) + if (i->first == "name") label = i->second; + } + + else if (ne.type == StoreExpr::neClosure) { + label = ""; + colour = "#00ffff"; + printClosure(path, ne); + } + + else abort(); + + cout << makeNode(path, label, colour); +#endif + } + + cout << "}\n"; +} + + +} diff --git a/third_party/nix/src/nix-store/dotgraph.hh b/third_party/nix/src/nix-store/dotgraph.hh new file mode 100644 index 000000000..e2b5fc72f --- /dev/null +++ b/third_party/nix/src/nix-store/dotgraph.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +class Store; + +void printDotGraph(ref store, const PathSet & roots); + +} diff --git a/third_party/nix/src/nix-store/graphml.cc b/third_party/nix/src/nix-store/graphml.cc new file mode 100644 index 000000000..670fbe227 --- /dev/null +++ b/third_party/nix/src/nix-store/graphml.cc @@ -0,0 +1,90 @@ +#include "graphml.hh" +#include "util.hh" +#include "store-api.hh" +#include "derivations.hh" + +#include + + +using std::cout; + +namespace nix { + + +static inline const string & xmlQuote(const string & s) +{ + // Luckily, store paths shouldn't contain any character that needs to be + // quoted. + return s; +} + + +static string symbolicName(const string & path) +{ + string p = baseNameOf(path); + return string(p, p.find('-') + 1); +} + + +static string makeEdge(const string & src, const string & dst) +{ + return fmt(" \n", + xmlQuote(src), xmlQuote(dst)); +} + + +static string makeNode(const ValidPathInfo & info) +{ + return fmt( + " \n" + " %2%\n" + " %3%\n" + " %4%\n" + " \n", + info.path, + info.narSize, + symbolicName(info.path), + (isDerivation(info.path) ? "derivation" : "output-path")); +} + + +void printGraphML(ref store, const PathSet & roots) +{ + PathSet workList(roots); + PathSet doneSet; + std::pair ret; + + cout << "\n" + << "\n" + << "" + << "" + << "" + << "\n"; + + while (!workList.empty()) { + Path path = *(workList.begin()); + workList.erase(path); + + ret = doneSet.insert(path); + if (ret.second == false) continue; + + ValidPathInfo info = *(store->queryPathInfo(path)); + cout << makeNode(info); + + for (auto & p : store->queryPathInfo(path)->references) { + if (p != path) { + workList.insert(p); + cout << makeEdge(path, p); + } + } + + } + + cout << "\n"; + cout << "\n"; +} + + +} diff --git a/third_party/nix/src/nix-store/graphml.hh b/third_party/nix/src/nix-store/graphml.hh new file mode 100644 index 000000000..b78df1e49 --- /dev/null +++ b/third_party/nix/src/nix-store/graphml.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "types.hh" + +namespace nix { + +class Store; + +void printGraphML(ref store, const PathSet & roots); + +} diff --git a/third_party/nix/src/nix-store/nix-store.cc b/third_party/nix/src/nix-store/nix-store.cc new file mode 100644 index 000000000..0cbceb02f --- /dev/null +++ b/third_party/nix/src/nix-store/nix-store.cc @@ -0,0 +1,1110 @@ +#include "archive.hh" +#include "derivations.hh" +#include "dotgraph.hh" +#include "globals.hh" +#include "local-store.hh" +#include "monitor-fd.hh" +#include "serve-protocol.hh" +#include "shared.hh" +#include "util.hh" +#include "worker-protocol.hh" +#include "graphml.hh" +#include "legacy.hh" + +#include +#include +#include + +#include +#include +#include + +#if HAVE_SODIUM +#include +#endif + + +using namespace nix; +using std::cin; +using std::cout; + + +typedef void (* Operation) (Strings opFlags, Strings opArgs); + + +static Path gcRoot; +static int rootNr = 0; +static bool indirectRoot = false; +static bool noOutput = false; +static std::shared_ptr store; + + +ref ensureLocalStore() +{ + auto store2 = std::dynamic_pointer_cast(store); + if (!store2) throw Error("you don't have sufficient rights to use this command"); + return ref(store2); +} + + +static Path useDeriver(Path path) +{ + if (isDerivation(path)) return path; + Path drvPath = store->queryPathInfo(path)->deriver; + if (drvPath == "") + throw Error(format("deriver of path '%1%' is not known") % path); + return drvPath; +} + + +/* Realise the given path. For a derivation that means build it; for + other paths it means ensure their validity. */ +static PathSet realisePath(Path path, bool build = true) +{ + DrvPathWithOutputs p = parseDrvPathWithOutputs(path); + + auto store2 = std::dynamic_pointer_cast(store); + + if (isDerivation(p.first)) { + if (build) store->buildPaths({path}); + Derivation drv = store->derivationFromPath(p.first); + rootNr++; + + if (p.second.empty()) + for (auto & i : drv.outputs) p.second.insert(i.first); + + PathSet outputs; + for (auto & j : p.second) { + DerivationOutputs::iterator i = drv.outputs.find(j); + if (i == drv.outputs.end()) + throw Error(format("derivation '%1%' does not have an output named '%2%'") % p.first % j); + Path outPath = i->second.path; + if (store2) { + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = gcRoot; + if (rootNr > 1) rootName += "-" + std::to_string(rootNr); + if (i->first != "out") rootName += "-" + i->first; + outPath = store2->addPermRoot(outPath, rootName, indirectRoot); + } + } + outputs.insert(outPath); + } + return outputs; + } + + else { + if (build) store->ensurePath(path); + else if (!store->isValidPath(path)) throw Error(format("path '%1%' does not exist and cannot be created") % path); + if (store2) { + if (gcRoot == "") + printGCWarning(); + else { + Path rootName = gcRoot; + rootNr++; + if (rootNr > 1) rootName += "-" + std::to_string(rootNr); + path = store2->addPermRoot(path, rootName, indirectRoot); + } + } + return {path}; + } +} + + +/* Realise the given paths. */ +static void opRealise(Strings opFlags, Strings opArgs) +{ + bool dryRun = false; + BuildMode buildMode = bmNormal; + bool ignoreUnknown = false; + + for (auto & i : opFlags) + if (i == "--dry-run") dryRun = true; + else if (i == "--repair") buildMode = bmRepair; + else if (i == "--check") buildMode = bmCheck; + else if (i == "--ignore-unknown") ignoreUnknown = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + Paths paths; + for (auto & i : opArgs) { + DrvPathWithOutputs p = parseDrvPathWithOutputs(i); + paths.push_back(makeDrvPathWithOutputs(store->followLinksToStorePath(p.first), p.second)); + } + + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(PathSet(paths.begin(), paths.end()), + willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (ignoreUnknown) { + Paths paths2; + for (auto & i : paths) + if (unknown.find(i) == unknown.end()) paths2.push_back(i); + paths = paths2; + unknown = PathSet(); + } + + if (settings.printMissing) + printMissing(ref(store), willBuild, willSubstitute, unknown, downloadSize, narSize); + + if (dryRun) return; + + /* Build all paths at the same time to exploit parallelism. */ + store->buildPaths(PathSet(paths.begin(), paths.end()), buildMode); + + if (!ignoreUnknown) + for (auto & i : paths) { + PathSet paths = realisePath(i, false); + if (!noOutput) + for (auto & j : paths) + cout << format("%1%\n") % j; + } +} + + +/* Add files to the Nix store and print the resulting paths. */ +static void opAdd(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + + for (auto & i : opArgs) + cout << format("%1%\n") % store->addToStore(baseNameOf(i), i); +} + + +/* Preload the output of a fixed-output derivation into the Nix + store. */ +static void opAddFixed(Strings opFlags, Strings opArgs) +{ + bool recursive = false; + + for (auto & i : opFlags) + if (i == "--recursive") recursive = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + if (opArgs.empty()) + throw UsageError("first argument must be hash algorithm"); + + HashType hashAlgo = parseHashType(opArgs.front()); + opArgs.pop_front(); + + for (auto & i : opArgs) + cout << format("%1%\n") % store->addToStore(baseNameOf(i), i, recursive, hashAlgo); +} + + +/* Hack to support caching in `nix-prefetch-url'. */ +static void opPrintFixedPath(Strings opFlags, Strings opArgs) +{ + bool recursive = false; + + for (auto i : opFlags) + if (i == "--recursive") recursive = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + if (opArgs.size() != 3) + throw UsageError(format("'--print-fixed-path' requires three arguments")); + + Strings::iterator i = opArgs.begin(); + HashType hashAlgo = parseHashType(*i++); + string hash = *i++; + string name = *i++; + + cout << format("%1%\n") % + store->makeFixedOutputPath(recursive, Hash(hash, hashAlgo), name); +} + + +static PathSet maybeUseOutputs(const Path & storePath, bool useOutput, bool forceRealise) +{ + if (forceRealise) realisePath(storePath); + if (useOutput && isDerivation(storePath)) { + Derivation drv = store->derivationFromPath(storePath); + PathSet outputs; + for (auto & i : drv.outputs) + outputs.insert(i.second.path); + return outputs; + } + else return {storePath}; +} + + +/* Some code to print a tree representation of a derivation dependency + graph. Topological sorting is used to keep the tree relatively + flat. */ + +const string treeConn = "+---"; +const string treeLine = "| "; +const string treeNull = " "; + + +static void printTree(const Path & path, + const string & firstPad, const string & tailPad, PathSet & done) +{ + if (done.find(path) != done.end()) { + cout << format("%1%%2% [...]\n") % firstPad % path; + return; + } + done.insert(path); + + cout << format("%1%%2%\n") % firstPad % path; + + auto references = store->queryPathInfo(path)->references; + + /* Topologically sort under the relation A < B iff A \in + closure(B). That is, if derivation A is an (possibly indirect) + input of B, then A is printed first. This has the effect of + flattening the tree, preventing deeply nested structures. */ + Paths sorted = store->topoSortPaths(references); + reverse(sorted.begin(), sorted.end()); + + for (auto i = sorted.begin(); i != sorted.end(); ++i) { + auto j = i; ++j; + printTree(*i, tailPad + treeConn, + j == sorted.end() ? tailPad + treeNull : tailPad + treeLine, + done); + } +} + + +/* Perform various sorts of queries. */ +static void opQuery(Strings opFlags, Strings opArgs) +{ + enum QueryType + { qDefault, qOutputs, qRequisites, qReferences, qReferrers + , qReferrersClosure, qDeriver, qBinding, qHash, qSize + , qTree, qGraph, qGraphML, qResolve, qRoots }; + QueryType query = qDefault; + bool useOutput = false; + bool includeOutputs = false; + bool forceRealise = false; + string bindingName; + + for (auto & i : opFlags) { + QueryType prev = query; + if (i == "--outputs") query = qOutputs; + else if (i == "--requisites" || i == "-R") query = qRequisites; + else if (i == "--references") query = qReferences; + else if (i == "--referrers" || i == "--referers") query = qReferrers; + else if (i == "--referrers-closure" || i == "--referers-closure") query = qReferrersClosure; + else if (i == "--deriver" || i == "-d") query = qDeriver; + else if (i == "--binding" || i == "-b") { + if (opArgs.size() == 0) + throw UsageError("expected binding name"); + bindingName = opArgs.front(); + opArgs.pop_front(); + query = qBinding; + } + else if (i == "--hash") query = qHash; + else if (i == "--size") query = qSize; + else if (i == "--tree") query = qTree; + else if (i == "--graph") query = qGraph; + else if (i == "--graphml") query = qGraphML; + else if (i == "--resolve") query = qResolve; + else if (i == "--roots") query = qRoots; + else if (i == "--use-output" || i == "-u") useOutput = true; + else if (i == "--force-realise" || i == "--force-realize" || i == "-f") forceRealise = true; + else if (i == "--include-outputs") includeOutputs = true; + else throw UsageError(format("unknown flag '%1%'") % i); + if (prev != qDefault && prev != query) + throw UsageError(format("query type '%1%' conflicts with earlier flag") % i); + } + + if (query == qDefault) query = qOutputs; + + RunPager pager; + + switch (query) { + + case qOutputs: { + for (auto & i : opArgs) { + i = store->followLinksToStorePath(i); + if (forceRealise) realisePath(i); + Derivation drv = store->derivationFromPath(i); + for (auto & j : drv.outputs) + cout << format("%1%\n") % j.second.path; + } + break; + } + + case qRequisites: + case qReferences: + case qReferrers: + case qReferrersClosure: { + PathSet paths; + for (auto & i : opArgs) { + PathSet ps = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); + for (auto & j : ps) { + if (query == qRequisites) store->computeFSClosure(j, paths, false, includeOutputs); + else if (query == qReferences) { + for (auto & p : store->queryPathInfo(j)->references) + paths.insert(p); + } + else if (query == qReferrers) store->queryReferrers(j, paths); + else if (query == qReferrersClosure) store->computeFSClosure(j, paths, true); + } + } + Paths sorted = store->topoSortPaths(paths); + for (Paths::reverse_iterator i = sorted.rbegin(); + i != sorted.rend(); ++i) + cout << format("%s\n") % *i; + break; + } + + case qDeriver: + for (auto & i : opArgs) { + Path deriver = store->queryPathInfo(store->followLinksToStorePath(i))->deriver; + cout << format("%1%\n") % + (deriver == "" ? "unknown-deriver" : deriver); + } + break; + + case qBinding: + for (auto & i : opArgs) { + Path path = useDeriver(store->followLinksToStorePath(i)); + Derivation drv = store->derivationFromPath(path); + StringPairs::iterator j = drv.env.find(bindingName); + if (j == drv.env.end()) + throw Error(format("derivation '%1%' has no environment binding named '%2%'") + % path % bindingName); + cout << format("%1%\n") % j->second; + } + break; + + case qHash: + case qSize: + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); + for (auto & j : paths) { + auto info = store->queryPathInfo(j); + if (query == qHash) { + assert(info->narHash.type == htSHA256); + cout << fmt("%s\n", info->narHash.to_string(Base32)); + } else if (query == qSize) + cout << fmt("%d\n", info->narSize); + } + } + break; + + case qTree: { + PathSet done; + for (auto & i : opArgs) + printTree(store->followLinksToStorePath(i), "", "", done); + break; + } + + case qGraph: { + PathSet roots; + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printDotGraph(ref(store), roots); + break; + } + + case qGraphML: { + PathSet roots; + for (auto & i : opArgs) { + PathSet paths = maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise); + roots.insert(paths.begin(), paths.end()); + } + printGraphML(ref(store), roots); + break; + } + + case qResolve: { + for (auto & i : opArgs) + cout << format("%1%\n") % store->followLinksToStorePath(i); + break; + } + + case qRoots: { + PathSet referrers; + for (auto & i : opArgs) { + store->computeFSClosure( + maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise), + referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations); + } + Roots roots = store->findRoots(false); + for (auto & [target, links] : roots) + if (referrers.find(target) != referrers.end()) + for (auto & link : links) + cout << format("%1% -> %2%\n") % link % target; + break; + } + + default: + abort(); + } +} + + +static void opPrintEnv(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("'--print-env' requires one derivation store path"); + + Path drvPath = opArgs.front(); + Derivation drv = store->derivationFromPath(drvPath); + + /* Print each environment variable in the derivation in a format + that can be sourced by the shell. */ + for (auto & i : drv.env) + cout << format("export %1%; %1%=%2%\n") % i.first % shellEscape(i.second); + + /* Also output the arguments. This doesn't preserve whitespace in + arguments. */ + cout << "export _args; _args='"; + bool first = true; + for (auto & i : drv.args) { + if (!first) cout << ' '; + first = false; + cout << shellEscape(i); + } + cout << "'\n"; +} + + +static void opReadLog(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + + RunPager pager; + + for (auto & i : opArgs) { + auto path = store->followLinksToStorePath(i); + auto log = store->getBuildLog(path); + if (!log) + throw Error("build log of derivation '%s' is not available", path); + std::cout << *log; + } +} + + +static void opDumpDB(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) { + for (auto & i : opArgs) + i = store->followLinksToStorePath(i); + for (auto & i : opArgs) + cout << store->makeValidityRegistration({i}, true, true); + } else { + PathSet validPaths = store->queryAllValidPaths(); + for (auto & i : validPaths) + cout << store->makeValidityRegistration({i}, true, true); + } +} + + +static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) +{ + ValidPathInfos infos; + + while (1) { + ValidPathInfo info = decodeValidPathInfo(cin, hashGiven); + if (info.path == "") break; + if (!store->isValidPath(info.path) || reregister) { + /* !!! races */ + if (canonicalise) + canonicalisePathMetaData(info.path, -1); + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.narHash = hash.first; + info.narSize = hash.second; + } + infos.push_back(info); + } + } + + ensureLocalStore()->registerValidPaths(infos); +} + + +static void opLoadDB(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + registerValidity(true, true, false); +} + + +static void opRegisterValidity(Strings opFlags, Strings opArgs) +{ + bool reregister = false; // !!! maybe this should be the default + bool hashGiven = false; + + for (auto & i : opFlags) + if (i == "--reregister") reregister = true; + else if (i == "--hash-given") hashGiven = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + registerValidity(reregister, hashGiven, true); +} + + +static void opCheckValidity(Strings opFlags, Strings opArgs) +{ + bool printInvalid = false; + + for (auto & i : opFlags) + if (i == "--print-invalid") printInvalid = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + for (auto & i : opArgs) { + Path path = store->followLinksToStorePath(i); + if (!store->isValidPath(path)) { + if (printInvalid) + cout << format("%1%\n") % path; + else + throw Error(format("path '%1%' is not valid") % path); + } + } +} + + +static void opGC(Strings opFlags, Strings opArgs) +{ + bool printRoots = false; + GCOptions options; + options.action = GCOptions::gcDeleteDead; + + GCResults results; + + /* Do what? */ + for (auto i = opFlags.begin(); i != opFlags.end(); ++i) + if (*i == "--print-roots") printRoots = true; + else if (*i == "--print-live") options.action = GCOptions::gcReturnLive; + else if (*i == "--print-dead") options.action = GCOptions::gcReturnDead; + else if (*i == "--delete") options.action = GCOptions::gcDeleteDead; + else if (*i == "--max-freed") { + long long maxFreed = getIntArg(*i, i, opFlags.end(), true); + options.maxFreed = maxFreed >= 0 ? maxFreed : 0; + } + else throw UsageError(format("bad sub-operation '%1%' in GC") % *i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + if (printRoots) { + Roots roots = store->findRoots(false); + std::set> roots2; + // Transpose and sort the roots. + for (auto & [target, links] : roots) + for (auto & link : links) + roots2.emplace(link, target); + for (auto & [link, target] : roots2) + std::cout << link << " -> " << target << "\n"; + } + + else { + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + + if (options.action != GCOptions::gcDeleteDead) + for (auto & i : results.paths) + cout << i << std::endl; + } +} + + +/* Remove paths from the Nix store if possible (i.e., if they do not + have any remaining referrers and are not reachable from any GC + roots). */ +static void opDelete(Strings opFlags, Strings opArgs) +{ + GCOptions options; + options.action = GCOptions::gcDeleteSpecific; + + for (auto & i : opFlags) + if (i == "--ignore-liveness") options.ignoreLiveness = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + for (auto & i : opArgs) + options.pathsToDelete.insert(store->followLinksToStorePath(i)); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); +} + + +/* Dump a path as a Nix archive. The archive is written to standard + output. */ +static void opDump(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("only one argument allowed"); + + FdSink sink(STDOUT_FILENO); + string path = *opArgs.begin(); + dumpPath(path, sink); + sink.flush(); +} + + +/* Restore a value from a Nix archive. The archive is read from + standard input. */ +static void opRestore(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (opArgs.size() != 1) throw UsageError("only one argument allowed"); + + FdSource source(STDIN_FILENO); + restorePath(*opArgs.begin(), source); +} + + +static void opExport(Strings opFlags, Strings opArgs) +{ + for (auto & i : opFlags) + throw UsageError(format("unknown flag '%1%'") % i); + + for (auto & i : opArgs) + i = store->followLinksToStorePath(i); + + FdSink sink(STDOUT_FILENO); + store->exportPaths(opArgs, sink); + sink.flush(); +} + + +static void opImport(Strings opFlags, Strings opArgs) +{ + for (auto & i : opFlags) + throw UsageError(format("unknown flag '%1%'") % i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + FdSource source(STDIN_FILENO); + Paths paths = store->importPaths(source, nullptr, NoCheckSigs); + + for (auto & i : paths) + cout << format("%1%\n") % i << std::flush; +} + + +/* Initialise the Nix databases. */ +static void opInit(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) throw UsageError("unknown flag"); + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + /* Doesn't do anything right now; database tables are initialised + automatically. */ +} + + +/* Verify the consistency of the Nix environment. */ +static void opVerify(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty()) + throw UsageError("no arguments expected"); + + bool checkContents = false; + RepairFlag repair = NoRepair; + + for (auto & i : opFlags) + if (i == "--check-contents") checkContents = true; + else if (i == "--repair") repair = Repair; + else throw UsageError(format("unknown flag '%1%'") % i); + + if (store->verifyStore(checkContents, repair)) { + printError("warning: not all errors were fixed"); + throw Exit(1); + } +} + + +/* Verify whether the contents of the given store path have not changed. */ +static void opVerifyPath(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + + int status = 0; + + for (auto & i : opArgs) { + Path path = store->followLinksToStorePath(i); + printMsg(lvlTalkative, format("checking path '%1%'...") % path); + auto info = store->queryPathInfo(path); + HashSink sink(info->narHash.type); + store->narFromPath(path, sink); + auto current = sink.finish(); + if (current.first != info->narHash) { + printError( + format("path '%1%' was modified! expected hash '%2%', got '%3%'") + % path % info->narHash.to_string() % current.first.to_string()); + status = 1; + } + } + + throw Exit(status); +} + + +/* Repair the contents of the given path by redownloading it using a + substituter (if available). */ +static void opRepairPath(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + + for (auto & i : opArgs) { + Path path = store->followLinksToStorePath(i); + ensureLocalStore()->repairPath(path); + } +} + +/* Optimise the disk space usage of the Nix store by hard-linking + files with the same contents. */ +static void opOptimise(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments expected"); + + store->optimiseStore(); +} + +/* Serve the nix store in a way usable by a restricted ssh user. */ +static void opServe(Strings opFlags, Strings opArgs) +{ + bool writeAllowed = false; + for (auto & i : opFlags) + if (i == "--write") writeAllowed = true; + else throw UsageError(format("unknown flag '%1%'") % i); + + if (!opArgs.empty()) throw UsageError("no arguments expected"); + + FdSource in(STDIN_FILENO); + FdSink out(STDOUT_FILENO); + + /* Exchange the greeting. */ + unsigned int magic = readInt(in); + if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); + out << SERVE_MAGIC_2 << SERVE_PROTOCOL_VERSION; + out.flush(); + unsigned int clientVersion = readInt(in); + + auto getBuildSettings = [&]() { + // FIXME: changing options here doesn't work if we're + // building through the daemon. + verbosity = lvlError; + settings.keepLog = false; + settings.useSubstitutes = false; + settings.maxSilentTime = readInt(in); + settings.buildTimeout = readInt(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 2) + settings.maxLogSize = readNum(in); + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { + settings.buildRepeat = readInt(in); + settings.enforceDeterminism = readInt(in); + settings.runDiffHook = true; + } + settings.printRepeatedBuilds = false; + }; + + while (true) { + ServeCommand cmd; + try { + cmd = (ServeCommand) readInt(in); + } catch (EndOfFile & e) { + break; + } + + switch (cmd) { + + case cmdQueryValidPaths: { + bool lock = readInt(in); + bool substitute = readInt(in); + PathSet paths = readStorePaths(*store, in); + if (lock && writeAllowed) + for (auto & path : paths) + store->addTempRoot(path); + + /* If requested, substitute missing paths. This + implements nix-copy-closure's --use-substitutes + flag. */ + if (substitute && writeAllowed) { + /* Filter out .drv files (we don't want to build anything). */ + PathSet paths2; + for (auto & path : paths) + if (!isDerivation(path)) paths2.insert(path); + unsigned long long downloadSize, narSize; + PathSet willBuild, willSubstitute, unknown; + store->queryMissing(PathSet(paths2.begin(), paths2.end()), + willBuild, willSubstitute, unknown, downloadSize, narSize); + /* FIXME: should use ensurePath(), but it only + does one path at a time. */ + if (!willSubstitute.empty()) + try { + store->buildPaths(willSubstitute); + } catch (Error & e) { + printError(format("warning: %1%") % e.msg()); + } + } + + out << store->queryValidPaths(paths); + break; + } + + case cmdQueryPathInfos: { + PathSet paths = readStorePaths(*store, in); + // !!! Maybe we want a queryPathInfos? + for (auto & i : paths) { + try { + auto info = store->queryPathInfo(i); + out << info->path << info->deriver << info->references; + // !!! Maybe we want compression? + out << info->narSize // downloadSize + << info->narSize; + if (GET_PROTOCOL_MINOR(clientVersion) >= 4) + out << (info->narHash ? info->narHash.to_string() : "") << info->ca << info->sigs; + } catch (InvalidPath &) { + } + } + out << ""; + break; + } + + case cmdDumpStorePath: + store->narFromPath(readStorePath(*store, in), out); + break; + + case cmdImportPaths: { + if (!writeAllowed) throw Error("importing paths is not allowed"); + store->importPaths(in, nullptr, NoCheckSigs); // FIXME: should we skip sig checking? + out << 1; // indicate success + break; + } + + case cmdExportPaths: { + readInt(in); // obsolete + store->exportPaths(readStorePaths(*store, in), out); + break; + } + + case cmdBuildPaths: { + + if (!writeAllowed) throw Error("building paths is not allowed"); + PathSet paths = readStorePaths(*store, in); + + getBuildSettings(); + + try { + MonitorFdHup monitor(in.fd); + store->buildPaths(paths); + out << 0; + } catch (Error & e) { + assert(e.status); + out << e.status << e.msg(); + } + break; + } + + case cmdBuildDerivation: { /* Used by hydra-queue-runner. */ + + if (!writeAllowed) throw Error("building paths is not allowed"); + + Path drvPath = readStorePath(*store, in); // informational only + BasicDerivation drv; + readDerivation(in, *store, drv); + + getBuildSettings(); + + MonitorFdHup monitor(in.fd); + auto status = store->buildDerivation(drvPath, drv); + + out << status.status << status.errorMsg; + + if (GET_PROTOCOL_MINOR(clientVersion) >= 3) + out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; + + break; + } + + case cmdQueryClosure: { + bool includeOutputs = readInt(in); + PathSet closure; + store->computeFSClosure(readStorePaths(*store, in), + closure, false, includeOutputs); + out << closure; + break; + } + + case cmdAddToStoreNar: { + if (!writeAllowed) throw Error("importing paths is not allowed"); + + ValidPathInfo info; + info.path = readStorePath(*store, in); + in >> info.deriver; + if (!info.deriver.empty()) + store->assertStorePath(info.deriver); + info.narHash = Hash(readString(in), htSHA256); + info.references = readStorePaths(*store, in); + in >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(in); + in >> info.ca; + + if (info.narSize == 0) { + throw Error("narInfo is too old and missing the narSize field"); + } + + SizedSource sizedSource(in, info.narSize); + + store->addToStore(info, sizedSource, NoRepair, NoCheckSigs); + + // consume all the data that has been sent before continuing. + sizedSource.drainAll(); + + out << 1; // indicate success + + break; + } + + default: + throw Error(format("unknown serve command %1%") % cmd); + } + + out.flush(); + } +} + + +static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) +{ + for (auto & i : opFlags) + throw UsageError(format("unknown flag '%1%'") % i); + + if (opArgs.size() != 3) throw UsageError("three arguments expected"); + auto i = opArgs.begin(); + string keyName = *i++; + string secretKeyFile = *i++; + string publicKeyFile = *i++; + +#if HAVE_SODIUM + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); + umask(0077); + writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); +#else + throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); +#endif +} + + +static void opVersion(Strings opFlags, Strings opArgs) +{ + printVersion("nix-store"); +} + + +/* Scan the arguments; find the operation, set global flags, put all + other flags in a list, and put all other arguments in another + list. */ +static int _main(int argc, char * * argv) +{ + { + Strings opFlags, opArgs; + Operation op = 0; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + Operation oldOp = op; + + if (*arg == "--help") + showManPage("nix-store"); + else if (*arg == "--version") + op = opVersion; + else if (*arg == "--realise" || *arg == "--realize" || *arg == "-r") + op = opRealise; + else if (*arg == "--add" || *arg == "-A") + op = opAdd; + else if (*arg == "--add-fixed") + op = opAddFixed; + else if (*arg == "--print-fixed-path") + op = opPrintFixedPath; + else if (*arg == "--delete") + op = opDelete; + else if (*arg == "--query" || *arg == "-q") + op = opQuery; + else if (*arg == "--print-env") + op = opPrintEnv; + else if (*arg == "--read-log" || *arg == "-l") + op = opReadLog; + else if (*arg == "--dump-db") + op = opDumpDB; + else if (*arg == "--load-db") + op = opLoadDB; + else if (*arg == "--register-validity") + op = opRegisterValidity; + else if (*arg == "--check-validity") + op = opCheckValidity; + else if (*arg == "--gc") + op = opGC; + else if (*arg == "--dump") + op = opDump; + else if (*arg == "--restore") + op = opRestore; + else if (*arg == "--export") + op = opExport; + else if (*arg == "--import") + op = opImport; + else if (*arg == "--init") + op = opInit; + else if (*arg == "--verify") + op = opVerify; + else if (*arg == "--verify-path") + op = opVerifyPath; + else if (*arg == "--repair-path") + op = opRepairPath; + else if (*arg == "--optimise" || *arg == "--optimize") + op = opOptimise; + else if (*arg == "--serve") + op = opServe; + else if (*arg == "--generate-binary-cache-key") + op = opGenerateBinaryCacheKey; + else if (*arg == "--add-root") + gcRoot = absPath(getArg(*arg, arg, end)); + else if (*arg == "--indirect") + indirectRoot = true; + else if (*arg == "--no-output") + noOutput = true; + else if (*arg != "" && arg->at(0) == '-') { + opFlags.push_back(*arg); + if (*arg == "--max-freed" || *arg == "--max-links" || *arg == "--max-atime") /* !!! hack */ + opFlags.push_back(getArg(*arg, arg, end)); + } + else + opArgs.push_back(*arg); + + if (oldOp && oldOp != op) + throw UsageError("only one operation may be specified"); + + return true; + }); + + initPlugins(); + + if (!op) throw UsageError("no operation specified"); + + if (op != opDump && op != opRestore) /* !!! hack */ + store = openStore(); + + op(opFlags, opArgs); + + return 0; + } +} + +static RegisterLegacyCommand s1("nix-store", _main); diff --git a/third_party/nix/src/nix/add-to-store.cc b/third_party/nix/src/nix/add-to-store.cc new file mode 100644 index 000000000..e86b96e3f --- /dev/null +++ b/third_party/nix/src/nix/add-to-store.cc @@ -0,0 +1,61 @@ +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" + +using namespace nix; + +struct CmdAddToStore : MixDryRun, StoreCommand +{ + Path path; + std::optional namePart; + + CmdAddToStore() + { + expectArg("path", &path); + + mkFlag() + .longName("name") + .shortName('n') + .description("name component of the store path") + .labels({"name"}) + .dest(&namePart); + } + + std::string name() override + { + return "add-to-store"; + } + + std::string description() override + { + return "add a path to the Nix store"; + } + + Examples examples() override + { + return { + }; + } + + void run(ref store) override + { + if (!namePart) namePart = baseNameOf(path); + + StringSink sink; + dumpPath(path, sink); + + ValidPathInfo info; + info.narHash = hashString(htSHA256, *sink.s); + info.narSize = sink.s->size(); + info.path = store->makeFixedOutputPath(true, info.narHash, *namePart); + info.ca = makeFixedOutputCA(true, info.narHash); + + if (!dryRun) + store->addToStore(info, sink.s); + + std::cout << fmt("%s\n", info.path); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/build.cc b/third_party/nix/src/nix/build.cc new file mode 100644 index 000000000..b329ac38a --- /dev/null +++ b/third_party/nix/src/nix/build.cc @@ -0,0 +1,72 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdBuild : MixDryRun, InstallablesCommand +{ + Path outLink = "result"; + + CmdBuild() + { + mkFlag() + .longName("out-link") + .shortName('o') + .description("path of the symlink to the build result") + .labels({"path"}) + .dest(&outLink); + + mkFlag() + .longName("no-link") + .description("do not create a symlink to the build result") + .set(&outLink, Path("")); + } + + std::string name() override + { + return "build"; + } + + std::string description() override + { + return "build a derivation or fetch a store path"; + } + + Examples examples() override + { + return { + Example{ + "To build and run GNU Hello from NixOS 17.03:", + "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" + }, + Example{ + "To build the build.x86_64-linux attribute from release.nix:", + "nix build -f release.nix build.x86_64-linux" + }, + }; + } + + void run(ref store) override + { + auto buildables = build(store, dryRun ? DryRun : Build, installables); + + if (dryRun) return; + + for (size_t i = 0; i < buildables.size(); ++i) { + auto & b(buildables[i]); + + if (outLink != "") + for (auto & output : b.outputs) + if (auto store2 = store.dynamic_pointer_cast()) { + std::string symlink = outLink; + if (i) symlink += fmt("-%d", i); + if (output.first != "out") symlink += fmt("-%s", output.first); + store2->addPermRoot(output.second, absPath(symlink), true); + } + } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/cat.cc b/third_party/nix/src/nix/cat.cc new file mode 100644 index 000000000..a35f640d8 --- /dev/null +++ b/third_party/nix/src/nix/cat.cc @@ -0,0 +1,74 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" + +using namespace nix; + +struct MixCat : virtual Args +{ + std::string path; + + void cat(ref accessor) + { + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path '%1%' does not exist") % path); + if (st.type != FSAccessor::Type::tRegular) + throw Error(format("path '%1%' is not a regular file") % path); + + std::cout << accessor->readFile(path); + } +}; + +struct CmdCatStore : StoreCommand, MixCat +{ + CmdCatStore() + { + expectArg("path", &path); + } + + std::string name() override + { + return "cat-store"; + } + + std::string description() override + { + return "print the contents of a store file on stdout"; + } + + void run(ref store) override + { + cat(store->getFSAccessor()); + } +}; + +struct CmdCatNar : StoreCommand, MixCat +{ + Path narPath; + + CmdCatNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + std::string name() override + { + return "cat-nar"; + } + + std::string description() override + { + return "print the contents of a file inside a NAR file"; + } + + void run(ref store) override + { + cat(makeNarAccessor(make_ref(readFile(narPath)))); + } +}; + +static RegisterCommand r1(make_ref()); +static RegisterCommand r2(make_ref()); diff --git a/third_party/nix/src/nix/command.cc b/third_party/nix/src/nix/command.cc new file mode 100644 index 000000000..3d7d582d6 --- /dev/null +++ b/third_party/nix/src/nix/command.cc @@ -0,0 +1,156 @@ +#include "command.hh" +#include "store-api.hh" +#include "derivations.hh" + +namespace nix { + +Commands * RegisterCommand::commands = 0; + +void Command::printHelp(const string & programName, std::ostream & out) +{ + Args::printHelp(programName, out); + + auto exs = examples(); + if (!exs.empty()) { + out << "\n"; + out << "Examples:\n"; + for (auto & ex : exs) + out << "\n" + << " " << ex.description << "\n" // FIXME: wrap + << " $ " << ex.command << "\n"; + } +} + +MultiCommand::MultiCommand(const Commands & _commands) + : commands(_commands) +{ + expectedArgs.push_back(ExpectedArg{"command", 1, true, [=](std::vector ss) { + assert(!command); + auto i = commands.find(ss[0]); + if (i == commands.end()) + throw UsageError("'%s' is not a recognised command", ss[0]); + command = i->second; + }}); +} + +void MultiCommand::printHelp(const string & programName, std::ostream & out) +{ + if (command) { + command->printHelp(programName + " " + command->name(), out); + return; + } + + out << "Usage: " << programName << " ... ...\n"; + + out << "\n"; + out << "Common flags:\n"; + printFlags(out); + + out << "\n"; + out << "Available commands:\n"; + + Table2 table; + for (auto & command : commands) { + auto descr = command.second->description(); + if (!descr.empty()) + table.push_back(std::make_pair(command.second->name(), descr)); + } + printTable(out, table); + +#if 0 + out << "\n"; + out << "For full documentation, run 'man " << programName << "' or 'man " << programName << "-'.\n"; +#endif +} + +bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end) +{ + if (Args::processFlag(pos, end)) return true; + if (command && command->processFlag(pos, end)) return true; + return false; +} + +bool MultiCommand::processArgs(const Strings & args, bool finish) +{ + if (command) + return command->processArgs(args, finish); + else + return Args::processArgs(args, finish); +} + +StoreCommand::StoreCommand() +{ +} + +ref StoreCommand::getStore() +{ + if (!_store) + _store = createStore(); + return ref(_store); +} + +ref StoreCommand::createStore() +{ + return openStore(); +} + +void StoreCommand::run() +{ + run(getStore()); +} + +StorePathsCommand::StorePathsCommand(bool recursive) + : recursive(recursive) +{ + if (recursive) + mkFlag() + .longName("no-recursive") + .description("apply operation to specified paths only") + .set(&this->recursive, false); + else + mkFlag() + .longName("recursive") + .shortName('r') + .description("apply operation to closure of the specified paths") + .set(&this->recursive, true); + + mkFlag(0, "all", "apply operation to the entire store", &all); +} + +void StorePathsCommand::run(ref store) +{ + Paths storePaths; + + if (all) { + if (installables.size()) + throw UsageError("'--all' does not expect arguments"); + for (auto & p : store->queryAllValidPaths()) + storePaths.push_back(p); + } + + else { + for (auto & p : toStorePaths(store, NoBuild, installables)) + storePaths.push_back(p); + + if (recursive) { + PathSet closure; + store->computeFSClosure(PathSet(storePaths.begin(), storePaths.end()), + closure, false, false); + storePaths = Paths(closure.begin(), closure.end()); + } + } + + run(store, storePaths); +} + +void StorePathCommand::run(ref store) +{ + auto storePaths = toStorePaths(store, NoBuild, installables); + + if (storePaths.size() != 1) + throw UsageError("this command requires exactly one store path"); + + run(store, *storePaths.begin()); +} + +} diff --git a/third_party/nix/src/nix/command.hh b/third_party/nix/src/nix/command.hh new file mode 100644 index 000000000..97a6fee7f --- /dev/null +++ b/third_party/nix/src/nix/command.hh @@ -0,0 +1,214 @@ +#pragma once + +#include "args.hh" +#include "common-eval-args.hh" + +namespace nix { + +extern std::string programPath; + +struct Value; +class Bindings; +class EvalState; + +/* A command is an argument parser that can be executed by calling its + run() method. */ +struct Command : virtual Args +{ + virtual std::string name() = 0; + virtual void prepare() { }; + virtual void run() = 0; + + struct Example + { + std::string description; + std::string command; + }; + + typedef std::list Examples; + + virtual Examples examples() { return Examples(); } + + void printHelp(const string & programName, std::ostream & out) override; +}; + +class Store; + +/* A command that require a Nix store. */ +struct StoreCommand : virtual Command +{ + StoreCommand(); + void run() override; + ref getStore(); + virtual ref createStore(); + virtual void run(ref) = 0; + +private: + std::shared_ptr _store; +}; + +struct Buildable +{ + Path drvPath; // may be empty + std::map outputs; +}; + +typedef std::vector Buildables; + +struct Installable +{ + virtual std::string what() = 0; + + virtual Buildables toBuildables() + { + throw Error("argument '%s' cannot be built", what()); + } + + Buildable toBuildable(); + + virtual Value * toValue(EvalState & state) + { + throw Error("argument '%s' cannot be evaluated", what()); + } +}; + +struct SourceExprCommand : virtual Args, StoreCommand, MixEvalArgs +{ + Path file; + + SourceExprCommand(); + + /* Return a value representing the Nix expression from which we + are installing. This is either the file specified by ‘--file’, + or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs + = import ...; bla = import ...; }’. */ + Value * getSourceExpr(EvalState & state); + + ref getEvalState(); + +private: + + std::shared_ptr evalState; + + Value * vSourceExpr = 0; +}; + +enum RealiseMode { Build, NoBuild, DryRun }; + +/* A command that operates on a list of "installables", which can be + store paths, attribute paths, Nix expressions, etc. */ +struct InstallablesCommand : virtual Args, SourceExprCommand +{ + std::vector> installables; + + InstallablesCommand() + { + expectArgs("installables", &_installables); + } + + void prepare() override; + + virtual bool useDefaultInstallables() { return true; } + +private: + + std::vector _installables; +}; + +struct InstallableCommand : virtual Args, SourceExprCommand +{ + std::shared_ptr installable; + + InstallableCommand() + { + expectArg("installable", &_installable); + } + + void prepare() override; + +private: + + std::string _installable; +}; + +/* A command that operates on zero or more store paths. */ +struct StorePathsCommand : public InstallablesCommand +{ +private: + + bool recursive = false; + bool all = false; + +public: + + StorePathsCommand(bool recursive = false); + + using StoreCommand::run; + + virtual void run(ref store, Paths storePaths) = 0; + + void run(ref store) override; + + bool useDefaultInstallables() override { return !all; } +}; + +/* A command that operates on exactly one store path. */ +struct StorePathCommand : public InstallablesCommand +{ + using StoreCommand::run; + + virtual void run(ref store, const Path & storePath) = 0; + + void run(ref store) override; +}; + +typedef std::map> Commands; + +/* An argument parser that supports multiple subcommands, + i.e. ‘ ’. */ +class MultiCommand : virtual Args +{ +public: + Commands commands; + + std::shared_ptr command; + + MultiCommand(const Commands & commands); + + void printHelp(const string & programName, std::ostream & out) override; + + bool processFlag(Strings::iterator & pos, Strings::iterator end) override; + + bool processArgs(const Strings & args, bool finish) override; +}; + +/* A helper class for registering commands globally. */ +struct RegisterCommand +{ + static Commands * commands; + + RegisterCommand(ref command) + { + if (!commands) commands = new Commands; + commands->emplace(command->name(), command); + } +}; + +std::shared_ptr parseInstallable( + SourceExprCommand & cmd, ref store, const std::string & installable, + bool useDefaultInstallables); + +Buildables build(ref store, RealiseMode mode, + std::vector> installables); + +PathSet toStorePaths(ref store, RealiseMode mode, + std::vector> installables); + +Path toStorePath(ref store, RealiseMode mode, + std::shared_ptr installable); + +PathSet toDerivations(ref store, + std::vector> installables, + bool useDeriver = false); + +} diff --git a/third_party/nix/src/nix/copy.cc b/third_party/nix/src/nix/copy.cc new file mode 100644 index 000000000..12a9f9cd3 --- /dev/null +++ b/third_party/nix/src/nix/copy.cc @@ -0,0 +1,100 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include + +using namespace nix; + +struct CmdCopy : StorePathsCommand +{ + std::string srcUri, dstUri; + + CheckSigsFlag checkSigs = CheckSigs; + + SubstituteFlag substitute = NoSubstitute; + + CmdCopy() + : StorePathsCommand(true) + { + mkFlag() + .longName("from") + .labels({"store-uri"}) + .description("URI of the source Nix store") + .dest(&srcUri); + mkFlag() + .longName("to") + .labels({"store-uri"}) + .description("URI of the destination Nix store") + .dest(&dstUri); + + mkFlag() + .longName("no-check-sigs") + .description("do not require that paths are signed by trusted keys") + .set(&checkSigs, NoCheckSigs); + + mkFlag() + .longName("substitute-on-destination") + .shortName('s') + .description("whether to try substitutes on the destination store (only supported by SSH)") + .set(&substitute, Substitute); + } + + std::string name() override + { + return "copy"; + } + + std::string description() override + { + return "copy paths between Nix stores"; + } + + Examples examples() override + { + return { + Example{ + "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", + "nix copy --to file:///tmp/cache $(type -p firefox)" + }, + Example{ + "To copy the entire current NixOS system closure to another machine via SSH:", + "nix copy --to ssh://server /run/current-system" + }, + Example{ + "To copy a closure from another machine via SSH:", + "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" + }, +#ifdef ENABLE_S3 + Example{ + "To copy Hello to an S3 binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs.hello" + }, + Example{ + "To copy Hello to an S3-compatible binary cache:", + "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs.hello" + }, +#endif + }; + } + + ref createStore() override + { + return srcUri.empty() ? StoreCommand::createStore() : openStore(srcUri); + } + + void run(ref srcStore, Paths storePaths) override + { + if (srcUri.empty() && dstUri.empty()) + throw UsageError("you must pass '--from' and/or '--to'"); + + ref dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + + copyPaths(srcStore, dstStore, PathSet(storePaths.begin(), storePaths.end()), + NoRepair, checkSigs, substitute); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/doctor.cc b/third_party/nix/src/nix/doctor.cc new file mode 100644 index 000000000..7b5444619 --- /dev/null +++ b/third_party/nix/src/nix/doctor.cc @@ -0,0 +1,124 @@ +#include "command.hh" +#include "serve-protocol.hh" +#include "shared.hh" +#include "store-api.hh" +#include "worker-protocol.hh" + +using namespace nix; + +std::string formatProtocol(unsigned int proto) +{ + if (proto) { + auto major = GET_PROTOCOL_MAJOR(proto) >> 8; + auto minor = GET_PROTOCOL_MINOR(proto); + return (format("%1%.%2%") % major % minor).str(); + } + return "unknown"; +} + +struct CmdDoctor : StoreCommand +{ + bool success = true; + + std::string name() override + { + return "doctor"; + } + + std::string description() override + { + return "check your system for potential problems"; + } + + void run(ref store) override + { + std::cout << "Store uri: " << store->getUri() << std::endl; + std::cout << std::endl; + + auto type = getStoreType(); + + if (type < tOther) { + success &= checkNixInPath(); + success &= checkProfileRoots(store); + } + success &= checkStoreProtocol(store->getProtocol()); + + if (!success) + throw Exit(2); + } + + bool checkNixInPath() + { + PathSet dirs; + + for (auto & dir : tokenizeString(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) + dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + + if (dirs.size() != 1) { + std::cout << "Warning: multiple versions of nix found in PATH." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkProfileRoots(ref store) + { + PathSet dirs; + + for (auto & dir : tokenizeString(getEnv("PATH"), ":")) { + Path profileDir = dirOf(dir); + try { + Path userEnv = canonPath(profileDir, true); + + if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { + while (profileDir.find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + + if (profileDir.find("/profiles/") == std::string::npos) + dirs.insert(dir); + } + } catch (SysError &) {} + } + + if (!dirs.empty()) { + std::cout << "Warning: found profiles outside of " << settings.nixStateDir << "/profiles." << std::endl; + std::cout << "The generation this profile points to might not have a gcroot and could be" << std::endl; + std::cout << "garbage collected, resulting in broken symlinks." << std::endl; + std::cout << std::endl; + for (auto & dir : dirs) + std::cout << " " << dir << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } + + bool checkStoreProtocol(unsigned int storeProto) + { + unsigned int clientProto = GET_PROTOCOL_MAJOR(SERVE_PROTOCOL_VERSION) == GET_PROTOCOL_MAJOR(storeProto) + ? SERVE_PROTOCOL_VERSION + : PROTOCOL_VERSION; + + if (clientProto != storeProto) { + std::cout << "Warning: protocol version of this client does not match the store." << std::endl; + std::cout << "While this is not necessarily a problem it's recommended to keep the client in" << std::endl; + std::cout << "sync with the daemon." << std::endl; + std::cout << std::endl; + std::cout << "Client protocol: " << formatProtocol(clientProto) << std::endl; + std::cout << "Store protocol: " << formatProtocol(storeProto) << std::endl; + std::cout << std::endl; + return false; + } + + return true; + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/dump-path.cc b/third_party/nix/src/nix/dump-path.cc new file mode 100644 index 000000000..f411c0cb7 --- /dev/null +++ b/third_party/nix/src/nix/dump-path.cc @@ -0,0 +1,36 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdDumpPath : StorePathCommand +{ + std::string name() override + { + return "dump-path"; + } + + std::string description() override + { + return "dump a store path to stdout (in NAR format)"; + } + + Examples examples() override + { + return { + Example{ + "To get a NAR from the binary cache https://cache.nixos.org/:", + "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" + }, + }; + } + + void run(ref store, const Path & storePath) override + { + FdSink sink(STDOUT_FILENO); + store->narFromPath(storePath, sink); + sink.flush(); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/edit.cc b/third_party/nix/src/nix/edit.cc new file mode 100644 index 000000000..a6169f118 --- /dev/null +++ b/third_party/nix/src/nix/edit.cc @@ -0,0 +1,81 @@ +#include "command.hh" +#include "shared.hh" +#include "eval.hh" +#include "attr-path.hh" +#include "progress-bar.hh" + +#include + +using namespace nix; + +struct CmdEdit : InstallableCommand +{ + std::string name() override + { + return "edit"; + } + + std::string description() override + { + return "open the Nix expression of a Nix package in $EDITOR"; + } + + Examples examples() override + { + return { + Example{ + "To open the Nix expression of the GNU Hello package:", + "nix edit nixpkgs.hello" + }, + }; + } + + void run(ref store) override + { + auto state = getEvalState(); + + auto v = installable->toValue(*state); + + Value * v2; + try { + auto dummyArgs = state->allocBindings(0); + v2 = findAlongAttrPath(*state, "meta.position", *dummyArgs, *v); + } catch (Error &) { + throw Error("package '%s' has no source location information", installable->what()); + } + + auto pos = state->forceString(*v2); + debug("position is %s", pos); + + auto colon = pos.rfind(':'); + if (colon == std::string::npos) + throw Error("cannot parse meta.position attribute '%s'", pos); + + std::string filename(pos, 0, colon); + int lineno; + try { + lineno = std::stoi(std::string(pos, colon + 1)); + } catch (std::invalid_argument & e) { + throw Error("cannot parse line number '%s'", pos); + } + + auto editor = getEnv("EDITOR", "cat"); + + auto args = tokenizeString(editor); + + if (editor.find("emacs") != std::string::npos || + editor.find("nano") != std::string::npos || + editor.find("vim") != std::string::npos) + args.push_back(fmt("+%d", lineno)); + + args.push_back(filename); + + stopProgressBar(); + + execvp(args.front().c_str(), stringsToCharPtrs(args).data()); + + throw SysError("cannot run editor '%s'", editor); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/eval.cc b/third_party/nix/src/nix/eval.cc new file mode 100644 index 000000000..b7058361c --- /dev/null +++ b/third_party/nix/src/nix/eval.cc @@ -0,0 +1,77 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "eval.hh" +#include "json.hh" +#include "value-to-json.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct CmdEval : MixJSON, InstallableCommand +{ + bool raw = false; + + CmdEval() + { + mkFlag(0, "raw", "print strings unquoted", &raw); + } + + std::string name() override + { + return "eval"; + } + + std::string description() override + { + return "evaluate a Nix expression"; + } + + Examples examples() override + { + return { + Example{ + "To evaluate a Nix expression given on the command line:", + "nix eval '(1 + 2)'" + }, + Example{ + "To evaluate a Nix expression from a file or URI:", + "nix eval -f channel:nixos-17.09 hello.name" + }, + Example{ + "To get the current version of Nixpkgs:", + "nix eval --raw nixpkgs.lib.nixpkgsVersion" + }, + Example{ + "To print the store path of the Hello package:", + "nix eval --raw nixpkgs.hello" + }, + }; + } + + void run(ref store) override + { + if (raw && json) + throw UsageError("--raw and --json are mutually exclusive"); + + auto state = getEvalState(); + + auto v = installable->toValue(*state); + PathSet context; + + stopProgressBar(); + + if (raw) { + std::cout << state->coerceToString(noPos, *v, context); + } else if (json) { + JSONPlaceholder jsonOut(std::cout); + printValueAsJSON(*state, true, *v, jsonOut, context); + } else { + state->forceValueDeep(*v); + std::cout << *v << "\n"; + } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/hash.cc b/third_party/nix/src/nix/hash.cc new file mode 100644 index 000000000..af4105e28 --- /dev/null +++ b/third_party/nix/src/nix/hash.cc @@ -0,0 +1,151 @@ +#include "command.hh" +#include "hash.hh" +#include "legacy.hh" +#include "shared.hh" + +using namespace nix; + +struct CmdHash : Command +{ + enum Mode { mFile, mPath }; + Mode mode; + Base base = SRI; + bool truncate = false; + HashType ht = htSHA256; + std::vector paths; + + CmdHash(Mode mode) : mode(mode) + { + mkFlag(0, "sri", "print hash in SRI format", &base, SRI); + mkFlag(0, "base64", "print hash in base-64", &base, Base64); + mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); + mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag() + .longName("type") + .mkHashTypeFlag(&ht); + expectArgs("paths", &paths); + } + + std::string name() override + { + return mode == mFile ? "hash-file" : "hash-path"; + } + + std::string description() override + { + return mode == mFile + ? "print cryptographic hash of a regular file" + : "print cryptographic hash of the NAR serialisation of a path"; + } + + void run() override + { + for (auto path : paths) { + Hash h = mode == mFile ? hashFile(ht, path) : hashPath(ht, path).first; + if (truncate && h.hashSize > 20) h = compressHash(h, 20); + std::cout << format("%1%\n") % + h.to_string(base, base == SRI); + } + } +}; + +static RegisterCommand r1(make_ref(CmdHash::mFile)); +static RegisterCommand r2(make_ref(CmdHash::mPath)); + +struct CmdToBase : Command +{ + Base base; + HashType ht = htUnknown; + std::vector args; + + CmdToBase(Base base) : base(base) + { + mkFlag() + .longName("type") + .mkHashTypeFlag(&ht); + expectArgs("strings", &args); + } + + std::string name() override + { + return + base == Base16 ? "to-base16" : + base == Base32 ? "to-base32" : + base == Base64 ? "to-base64" : + "to-sri"; + } + + std::string description() override + { + return fmt("convert a hash to %s representation", + base == Base16 ? "base-16" : + base == Base32 ? "base-32" : + base == Base64 ? "base-64" : + "SRI"); + } + + void run() override + { + for (auto s : args) + std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); + } +}; + +static RegisterCommand r3(make_ref(Base16)); +static RegisterCommand r4(make_ref(Base32)); +static RegisterCommand r5(make_ref(Base64)); +static RegisterCommand r6(make_ref(SRI)); + +/* Legacy nix-hash command. */ +static int compatNixHash(int argc, char * * argv) +{ + HashType ht = htMD5; + bool flat = false; + bool base32 = false; + bool truncate = false; + enum { opHash, opTo32, opTo16 } op = opHash; + std::vector ss; + + parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-hash"); + else if (*arg == "--version") + printVersion("nix-hash"); + else if (*arg == "--flat") flat = true; + else if (*arg == "--base32") base32 = true; + else if (*arg == "--truncate") truncate = true; + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + if (ht == htUnknown) + throw UsageError(format("unknown hash type '%1%'") % s); + } + else if (*arg == "--to-base16") op = opTo16; + else if (*arg == "--to-base32") op = opTo32; + else if (*arg != "" && arg->at(0) == '-') + return false; + else + ss.push_back(*arg); + return true; + }); + + if (op == opHash) { + CmdHash cmd(flat ? CmdHash::mFile : CmdHash::mPath); + cmd.ht = ht; + cmd.base = base32 ? Base32 : Base16; + cmd.truncate = truncate; + cmd.paths = ss; + cmd.run(); + } + + else { + CmdToBase cmd(op == opTo32 ? Base32 : Base16); + cmd.args = ss; + cmd.ht = ht; + cmd.run(); + } + + return 0; +} + +static RegisterLegacyCommand s1("nix-hash", compatNixHash); diff --git a/third_party/nix/src/nix/installables.cc b/third_party/nix/src/nix/installables.cc new file mode 100644 index 000000000..0e8bba39d --- /dev/null +++ b/third_party/nix/src/nix/installables.cc @@ -0,0 +1,327 @@ +#include "command.hh" +#include "attr-path.hh" +#include "common-eval-args.hh" +#include "derivations.hh" +#include "eval-inline.hh" +#include "eval.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "shared.hh" + +#include + +namespace nix { + +SourceExprCommand::SourceExprCommand() +{ + mkFlag() + .shortName('f') + .longName("file") + .label("file") + .description("evaluate FILE rather than the default") + .dest(&file); +} + +Value * SourceExprCommand::getSourceExpr(EvalState & state) +{ + if (vSourceExpr) return vSourceExpr; + + auto sToplevel = state.symbols.create("_toplevel"); + + vSourceExpr = state.allocValue(); + + if (file != "") + state.evalFile(lookupFileArg(state, file), *vSourceExpr); + + else { + + /* Construct the installation source from $NIX_PATH. */ + + auto searchPath = state.getSearchPath(); + + state.mkAttrs(*vSourceExpr, 1024); + + mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); + + std::unordered_set seen; + + auto addEntry = [&](const std::string & name) { + if (name == "") return; + if (!seen.insert(name).second) return; + Value * v1 = state.allocValue(); + mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); + Value * v2 = state.allocValue(); + mkApp(*v2, *v1, mkString(*state.allocValue(), name)); + mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), + state.getBuiltin("import"), *v2); + }; + + for (auto & i : searchPath) + /* Hack to handle channels. */ + if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { + for (auto & j : readDirectory(i.second)) + if (j.name != "manifest.nix" + && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) + addEntry(j.name); + } else + addEntry(i.first); + + vSourceExpr->attrs->sort(); + } + + return vSourceExpr; +} + +ref SourceExprCommand::getEvalState() +{ + if (!evalState) + evalState = std::make_shared(searchPath, getStore()); + return ref(evalState); +} + +Buildable Installable::toBuildable() +{ + auto buildables = toBuildables(); + if (buildables.size() != 1) + throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size()); + return std::move(buildables[0]); +} + +struct InstallableStorePath : Installable +{ + Path storePath; + + InstallableStorePath(const Path & storePath) : storePath(storePath) { } + + std::string what() override { return storePath; } + + Buildables toBuildables() override + { + return {{isDerivation(storePath) ? storePath : "", {{"out", storePath}}}}; + } +}; + +struct InstallableValue : Installable +{ + SourceExprCommand & cmd; + + InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + + Buildables toBuildables() override + { + auto state = cmd.getEvalState(); + + auto v = toValue(*state); + + Bindings & autoArgs = *cmd.getAutoArgs(*state); + + DrvInfos drvs; + getDerivations(*state, *v, "", autoArgs, drvs, false); + + Buildables res; + + PathSet drvPaths; + + for (auto & drv : drvs) { + Buildable b{drv.queryDrvPath()}; + drvPaths.insert(b.drvPath); + + auto outputName = drv.queryOutputName(); + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", b.drvPath); + + b.outputs.emplace(outputName, drv.queryOutPath()); + + res.push_back(std::move(b)); + } + + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{*drvPaths.begin()}; + for (auto & b2 : res) + b.outputs.insert(b2.outputs.begin(), b2.outputs.end()); + return {b}; + } else + return res; + } +}; + +struct InstallableExpr : InstallableValue +{ + std::string text; + + InstallableExpr(SourceExprCommand & cmd, const std::string & text) + : InstallableValue(cmd), text(text) { } + + std::string what() override { return text; } + + Value * toValue(EvalState & state) override + { + auto v = state.allocValue(); + state.eval(state.parseExprFromString(text, absPath(".")), *v); + return v; + } +}; + +struct InstallableAttrPath : InstallableValue +{ + std::string attrPath; + + InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) + : InstallableValue(cmd), attrPath(attrPath) + { } + + std::string what() override { return attrPath; } + + Value * toValue(EvalState & state) override + { + auto source = cmd.getSourceExpr(state); + + Bindings & autoArgs = *cmd.getAutoArgs(state); + + Value * v = findAlongAttrPath(state, attrPath, autoArgs, *source); + state.forceValue(*v); + + return v; + } +}; + +// FIXME: extend +std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; +static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); + +static std::vector> parseInstallables( + SourceExprCommand & cmd, ref store, std::vector ss, bool useDefaultInstallables) +{ + std::vector> result; + + if (ss.empty() && useDefaultInstallables) { + if (cmd.file == "") + cmd.file = "."; + ss = {""}; + } + + for (auto & s : ss) { + + if (s.compare(0, 1, "(") == 0) + result.push_back(std::make_shared(cmd, s)); + + else if (s.find("/") != std::string::npos) { + + auto path = store->toStorePath(store->followLinksToStore(s)); + + if (store->isStorePath(path)) + result.push_back(std::make_shared(path)); + } + + else if (s == "" || std::regex_match(s, attrPathRegex)) + result.push_back(std::make_shared(cmd, s)); + + else + throw UsageError("don't know what to do with argument '%s'", s); + } + + return result; +} + +std::shared_ptr parseInstallable( + SourceExprCommand & cmd, ref store, const std::string & installable, + bool useDefaultInstallables) +{ + auto installables = parseInstallables(cmd, store, {installable}, false); + assert(installables.size() == 1); + return installables.front(); +} + +Buildables build(ref store, RealiseMode mode, + std::vector> installables) +{ + if (mode != Build) + settings.readOnlyMode = true; + + Buildables buildables; + + PathSet pathsToBuild; + + for (auto & i : installables) { + for (auto & b : i->toBuildables()) { + if (b.drvPath != "") { + StringSet outputNames; + for (auto & output : b.outputs) + outputNames.insert(output.first); + pathsToBuild.insert( + b.drvPath + "!" + concatStringsSep(",", outputNames)); + } else + for (auto & output : b.outputs) + pathsToBuild.insert(output.second); + buildables.push_back(std::move(b)); + } + } + + if (mode == DryRun) + printMissing(store, pathsToBuild, lvlError); + else if (mode == Build) + store->buildPaths(pathsToBuild); + + return buildables; +} + +PathSet toStorePaths(ref store, RealiseMode mode, + std::vector> installables) +{ + PathSet outPaths; + + for (auto & b : build(store, mode, installables)) + for (auto & output : b.outputs) + outPaths.insert(output.second); + + return outPaths; +} + +Path toStorePath(ref store, RealiseMode mode, + std::shared_ptr installable) +{ + auto paths = toStorePaths(store, mode, {installable}); + + if (paths.size() != 1) + throw Error("argument '%s' should evaluate to one store path", installable->what()); + + return *paths.begin(); +} + +PathSet toDerivations(ref store, + std::vector> installables, bool useDeriver) +{ + PathSet drvPaths; + + for (auto & i : installables) + for (auto & b : i->toBuildables()) { + if (b.drvPath.empty()) { + if (!useDeriver) + throw Error("argument '%s' did not evaluate to a derivation", i->what()); + for (auto & output : b.outputs) { + auto derivers = store->queryValidDerivers(output.second); + if (derivers.empty()) + throw Error("'%s' does not have a known deriver", i->what()); + // FIXME: use all derivers? + drvPaths.insert(*derivers.begin()); + } + } else + drvPaths.insert(b.drvPath); + } + + return drvPaths; +} + +void InstallablesCommand::prepare() +{ + installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); +} + +void InstallableCommand::prepare() +{ + installable = parseInstallable(*this, getStore(), _installable, false); +} + +} diff --git a/third_party/nix/src/nix/legacy.cc b/third_party/nix/src/nix/legacy.cc new file mode 100644 index 000000000..6df09ee37 --- /dev/null +++ b/third_party/nix/src/nix/legacy.cc @@ -0,0 +1,7 @@ +#include "legacy.hh" + +namespace nix { + +RegisterLegacyCommand::Commands * RegisterLegacyCommand::commands = 0; + +} diff --git a/third_party/nix/src/nix/legacy.hh b/third_party/nix/src/nix/legacy.hh new file mode 100644 index 000000000..f503b0da3 --- /dev/null +++ b/third_party/nix/src/nix/legacy.hh @@ -0,0 +1,23 @@ +#pragma once + +#include +#include +#include + +namespace nix { + +typedef std::function MainFunction; + +struct RegisterLegacyCommand +{ + typedef std::map Commands; + static Commands * commands; + + RegisterLegacyCommand(const std::string & name, MainFunction fun) + { + if (!commands) commands = new Commands; + (*commands)[name] = fun; + } +}; + +} diff --git a/third_party/nix/src/nix/local.mk b/third_party/nix/src/nix/local.mk new file mode 100644 index 000000000..c09efd1fc --- /dev/null +++ b/third_party/nix/src/nix/local.mk @@ -0,0 +1,25 @@ +programs += nix + +nix_DIR := $(d) + +nix_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard src/build-remote/*.cc) \ + $(wildcard src/nix-build/*.cc) \ + $(wildcard src/nix-channel/*.cc) \ + $(wildcard src/nix-collect-garbage/*.cc) \ + $(wildcard src/nix-copy-closure/*.cc) \ + $(wildcard src/nix-daemon/*.cc) \ + $(wildcard src/nix-env/*.cc) \ + $(wildcard src/nix-instantiate/*.cc) \ + $(wildcard src/nix-prefetch-url/*.cc) \ + $(wildcard src/nix-store/*.cc) \ + +nix_LIBS = libexpr libmain libstore libutil + +nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -lboost_context -lboost_thread -lboost_system + +$(foreach name, \ + nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ + $(eval $(call install-symlink, nix, $(bindir)/$(name)))) +$(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) diff --git a/third_party/nix/src/nix/log.cc b/third_party/nix/src/nix/log.cc new file mode 100644 index 000000000..f07ec4e93 --- /dev/null +++ b/third_party/nix/src/nix/log.cc @@ -0,0 +1,71 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct CmdLog : InstallableCommand +{ + CmdLog() + { + } + + std::string name() override + { + return "log"; + } + + std::string description() override + { + return "show the build log of the specified packages or paths, if available"; + } + + Examples examples() override + { + return { + Example{ + "To get the build log of GNU Hello:", + "nix log nixpkgs.hello" + }, + Example{ + "To get the build log of a specific path:", + "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" + }, + Example{ + "To get a build log from a specific binary cache:", + "nix log --store https://cache.nixos.org nixpkgs.hello" + }, + }; + } + + void run(ref store) override + { + settings.readOnlyMode = true; + + auto subs = getDefaultSubstituters(); + + subs.push_front(store); + + auto b = installable->toBuildable(); + + RunPager pager; + for (auto & sub : subs) { + auto log = b.drvPath != "" ? sub->getBuildLog(b.drvPath) : nullptr; + for (auto & output : b.outputs) { + if (log) break; + log = sub->getBuildLog(output.second); + } + if (!log) continue; + stopProgressBar(); + printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri()); + std::cout << *log; + return; + } + + throw Error("build log of '%s' is not available", installable->what()); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/ls.cc b/third_party/nix/src/nix/ls.cc new file mode 100644 index 000000000..d089be42f --- /dev/null +++ b/third_party/nix/src/nix/ls.cc @@ -0,0 +1,156 @@ +#include "command.hh" +#include "store-api.hh" +#include "fs-accessor.hh" +#include "nar-accessor.hh" +#include "common-args.hh" +#include "json.hh" + +using namespace nix; + +struct MixLs : virtual Args, MixJSON +{ + std::string path; + + bool recursive = false; + bool verbose = false; + bool showDirectory = false; + + MixLs() + { + mkFlag('R', "recursive", "list subdirectories recursively", &recursive); + mkFlag('l', "long", "show more file information", &verbose); + mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + } + + void listText(ref accessor) + { + std::function doPath; + + auto showFile = [&](const Path & curPath, const std::string & relPath) { + if (verbose) { + auto st = accessor->stat(curPath); + std::string tp = + st.type == FSAccessor::Type::tRegular ? + (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : + st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + "dr-xr-xr-x"; + std::cout << + (format("%s %20d %s") % tp % st.fileSize % relPath); + if (st.type == FSAccessor::Type::tSymlink) + std::cout << " -> " << accessor->readLink(curPath) + ; + std::cout << "\n"; + if (recursive && st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } else { + std::cout << relPath << "\n"; + if (recursive) { + auto st = accessor->stat(curPath); + if (st.type == FSAccessor::Type::tDirectory) + doPath(st, curPath, relPath, false); + } + } + }; + + doPath = [&](const FSAccessor::Stat & st, const Path & curPath, + const std::string & relPath, bool showDirectory) + { + if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + auto names = accessor->readDirectory(curPath); + for (auto & name : names) + showFile(curPath + "/" + name, relPath + "/" + name); + } else + showFile(curPath, relPath); + }; + + auto st = accessor->stat(path); + if (st.type == FSAccessor::Type::tMissing) + throw Error(format("path '%1%' does not exist") % path); + doPath(st, path, + st.type == FSAccessor::Type::tDirectory ? "." : baseNameOf(path), + showDirectory); + } + + void list(ref accessor) + { + if (path == "/") path = ""; + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + listNar(jsonRoot, accessor, path, recursive); + } else + listText(accessor); + } +}; + +struct CmdLsStore : StoreCommand, MixLs +{ + CmdLsStore() + { + expectArg("path", &path); + } + + Examples examples() override + { + return { + Example{ + "To list the contents of a store path in a binary cache:", + "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" + }, + }; + } + + std::string name() override + { + return "ls-store"; + } + + std::string description() override + { + return "show information about a store path"; + } + + void run(ref store) override + { + list(store->getFSAccessor()); + } +}; + +struct CmdLsNar : Command, MixLs +{ + Path narPath; + + CmdLsNar() + { + expectArg("nar", &narPath); + expectArg("path", &path); + } + + Examples examples() override + { + return { + Example{ + "To list a specific file in a NAR:", + "nix ls-nar -l hello.nar /bin/hello" + }, + }; + } + + std::string name() override + { + return "ls-nar"; + } + + std::string description() override + { + return "show information about the contents of a NAR file"; + } + + void run() override + { + list(makeNarAccessor(make_ref(readFile(narPath, true)))); + } +}; + +static RegisterCommand r1(make_ref()); +static RegisterCommand r2(make_ref()); diff --git a/third_party/nix/src/nix/main.cc b/third_party/nix/src/nix/main.cc new file mode 100644 index 000000000..c683d86df --- /dev/null +++ b/third_party/nix/src/nix/main.cc @@ -0,0 +1,178 @@ +#include + +#include "command.hh" +#include "common-args.hh" +#include "eval.hh" +#include "globals.hh" +#include "legacy.hh" +#include "shared.hh" +#include "store-api.hh" +#include "progress-bar.hh" +#include "download.hh" +#include "finally.hh" + +#include +#include +#include +#include +#include + +extern std::string chrootHelperName; + +void chrootHelper(int argc, char * * argv); + +namespace nix { + +/* Check if we have a non-loopback/link-local network interface. */ +static bool haveInternet() +{ + struct ifaddrs * addrs; + + if (getifaddrs(&addrs)) + return true; + + Finally free([&]() { freeifaddrs(addrs); }); + + for (auto i = addrs; i; i = i->ifa_next) { + if (!i->ifa_addr) continue; + if (i->ifa_addr->sa_family == AF_INET) { + if (ntohl(((sockaddr_in *) i->ifa_addr)->sin_addr.s_addr) != INADDR_LOOPBACK) { + return true; + } + } else if (i->ifa_addr->sa_family == AF_INET6) { + if (!IN6_IS_ADDR_LOOPBACK(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr) && + !IN6_IS_ADDR_LINKLOCAL(&((sockaddr_in6 *) i->ifa_addr)->sin6_addr)) + return true; + } + } + + return false; +} + +std::string programPath; + +struct NixArgs : virtual MultiCommand, virtual MixCommonArgs +{ + bool printBuildLogs = false; + bool useNet = true; + + NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix") + { + mkFlag() + .longName("help") + .description("show usage information") + .handler([&]() { showHelpAndExit(); }); + + mkFlag() + .longName("help-config") + .description("show configuration options") + .handler([&]() { + std::cout << "The following configuration options are available:\n\n"; + Table2 tbl; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + tbl.emplace_back(s.first, s.second.description); + printTable(std::cout, tbl); + throw Exit(); + }); + + mkFlag() + .longName("print-build-logs") + .shortName('L') + .description("print full build logs on stderr") + .set(&printBuildLogs, true); + + mkFlag() + .longName("version") + .description("show version information") + .handler([&]() { printVersion(programName); }); + + mkFlag() + .longName("no-net") + .description("disable substituters and consider all previously downloaded files up-to-date") + .handler([&]() { useNet = false; }); + } + + void printFlags(std::ostream & out) override + { + Args::printFlags(out); + std::cout << + "\n" + "In addition, most configuration settings can be overriden using '-- '.\n" + "Boolean settings can be overriden using '--' or '--no-'. See 'nix\n" + "--help-config' for a list of configuration settings.\n"; + } + + void showHelpAndExit() + { + printHelp(programName, std::cout); + std::cout << "\nNote: this program is EXPERIMENTAL and subject to change.\n"; + throw Exit(); + } +}; + +void mainWrapped(int argc, char * * argv) +{ + /* The chroot helper needs to be run before any threads have been + started. */ + if (argc > 0 && argv[0] == chrootHelperName) { + chrootHelper(argc, argv); + return; + } + + initNix(); + initGC(); + + programPath = argv[0]; + string programName = baseNameOf(programPath); + + { + auto legacy = (*RegisterLegacyCommand::commands)[programName]; + if (legacy) return legacy(argc, argv); + } + + verbosity = lvlWarn; + settings.verboseBuild = false; + + NixArgs args; + + args.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (!args.command) args.showHelpAndExit(); + + Finally f([]() { stopProgressBar(); }); + + startProgressBar(args.printBuildLogs); + + if (args.useNet && !haveInternet()) { + warn("you don't have Internet access; disabling some network-dependent features"); + args.useNet = false; + } + + if (!args.useNet) { + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overriden) + settings.useSubstitutes = false; + if (!settings.tarballTtl.overriden) + settings.tarballTtl = std::numeric_limits::max(); + if (!downloadSettings.tries.overriden) + downloadSettings.tries = 0; + if (!downloadSettings.connectTimeout.overriden) + downloadSettings.connectTimeout = 1; + } + + args.command->prepare(); + args.command->run(); +} + +} + +int main(int argc, char * * argv) +{ + return nix::handleExceptions(argv[0], [&]() { + nix::mainWrapped(argc, argv); + }); +} diff --git a/third_party/nix/src/nix/optimise-store.cc b/third_party/nix/src/nix/optimise-store.cc new file mode 100644 index 000000000..725fb75a1 --- /dev/null +++ b/third_party/nix/src/nix/optimise-store.cc @@ -0,0 +1,41 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +#include + +using namespace nix; + +struct CmdOptimiseStore : StoreCommand +{ + CmdOptimiseStore() + { + } + + std::string name() override + { + return "optimise-store"; + } + + std::string description() override + { + return "replace identical files in the store by hard links"; + } + + Examples examples() override + { + return { + Example{ + "To optimise the Nix store:", + "nix optimise-store" + }, + }; + } + + void run(ref store) override + { + store->optimiseStore(); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/path-info.cc b/third_party/nix/src/nix/path-info.cc new file mode 100644 index 000000000..dea5f0557 --- /dev/null +++ b/third_party/nix/src/nix/path-info.cc @@ -0,0 +1,133 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "json.hh" +#include "common-args.hh" + +#include +#include + +using namespace nix; + +struct CmdPathInfo : StorePathsCommand, MixJSON +{ + bool showSize = false; + bool showClosureSize = false; + bool humanReadable = false; + bool showSigs = false; + + CmdPathInfo() + { + mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); + mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); + mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); + mkFlag(0, "sigs", "show signatures", &showSigs); + } + + std::string name() override + { + return "path-info"; + } + + std::string description() override + { + return "query information about store paths"; + } + + Examples examples() override + { + return { + Example{ + "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", + "nix path-info -rS /run/current-system | sort -nk2" + }, + Example{ + "To show a package's closure size and all its dependencies with human readable sizes:", + "nix path-info -rsSh nixpkgs.rust" + }, + Example{ + "To check the existence of a path in a binary cache:", + "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" + }, + Example{ + "To print the 10 most recently added paths (using --json and the jq(1) command):", + "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'" + }, + Example{ + "To show the size of the entire Nix store:", + "nix path-info --json --all | jq 'map(.narSize) | add'" + }, + Example{ + "To show every path whose closure is bigger than 1 GB, sorted by closure size:", + "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'" + }, + }; + } + + void printSize(unsigned long long value) + { + if (!humanReadable) { + std::cout << fmt("\t%11d", value); + return; + } + + static const std::array idents{{ + ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' + }}; + size_t power = 0; + double res = value; + while (res > 1024 && power < idents.size()) { + ++power; + res /= 1024; + } + std::cout << fmt("\t%6.1f%c", res, idents.at(power)); + } + + void run(ref store, Paths storePaths) override + { + size_t pathLen = 0; + for (auto & storePath : storePaths) + pathLen = std::max(pathLen, storePath.size()); + + if (json) { + JSONPlaceholder jsonRoot(std::cout); + store->pathInfoToJSON(jsonRoot, + // FIXME: preserve order? + PathSet(storePaths.begin(), storePaths.end()), + true, showClosureSize, AllowInvalid); + } + + else { + + for (auto storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + storePath = info->path; // FIXME: screws up padding + + std::cout << storePath; + + if (showSize || showClosureSize || showSigs) + std::cout << std::string(std::max(0, (int) pathLen - (int) storePath.size()), ' '); + + if (showSize) + printSize(info->narSize); + + if (showClosureSize) + printSize(store->getClosureSize(storePath).first); + + if (showSigs) { + std::cout << '\t'; + Strings ss; + if (info->ultimate) ss.push_back("ultimate"); + if (info->ca != "") ss.push_back("ca:" + info->ca); + for (auto & sig : info->sigs) ss.push_back(sig); + std::cout << concatStringsSep(" ", ss); + } + + std::cout << std::endl; + } + + } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/ping-store.cc b/third_party/nix/src/nix/ping-store.cc new file mode 100644 index 000000000..310942574 --- /dev/null +++ b/third_party/nix/src/nix/ping-store.cc @@ -0,0 +1,35 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdPingStore : StoreCommand +{ + std::string name() override + { + return "ping-store"; + } + + std::string description() override + { + return "test whether a store can be opened"; + } + + Examples examples() override + { + return { + Example{ + "To test whether connecting to a remote Nix store via SSH works:", + "nix ping-store --store ssh://mac1" + }, + }; + } + + void run(ref store) override + { + store->connect(); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/progress-bar.cc b/third_party/nix/src/nix/progress-bar.cc new file mode 100644 index 000000000..98049b5da --- /dev/null +++ b/third_party/nix/src/nix/progress-bar.cc @@ -0,0 +1,454 @@ +#include "progress-bar.hh" +#include "util.hh" +#include "sync.hh" +#include "store-api.hh" +#include "names.hh" + +#include +#include +#include + +namespace nix { + +static std::string getS(const std::vector & fields, size_t n) +{ + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tString); + return fields[n].s; +} + +static uint64_t getI(const std::vector & fields, size_t n) +{ + assert(n < fields.size()); + assert(fields[n].type == Logger::Field::tInt); + return fields[n].i; +} + +class ProgressBar : public Logger +{ +private: + + struct ActInfo + { + std::string s, lastLine, phase; + ActivityType type = actUnknown; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t running = 0; + uint64_t failed = 0; + std::map expectedByType; + bool visible = true; + ActivityId parent; + std::optional name; + }; + + struct ActivitiesByType + { + std::map::iterator> its; + uint64_t done = 0; + uint64_t expected = 0; + uint64_t failed = 0; + }; + + struct State + { + std::list activities; + std::map::iterator> its; + + std::map activitiesByType; + + uint64_t filesLinked = 0, bytesLinked = 0; + + uint64_t corruptedPaths = 0, untrustedPaths = 0; + + bool active = true; + bool haveUpdate = true; + }; + + Sync state_; + + std::thread updateThread; + + std::condition_variable quitCV, updateCV; + + bool printBuildLogs; + bool isTTY; + +public: + + ProgressBar(bool printBuildLogs, bool isTTY) + : printBuildLogs(printBuildLogs) + , isTTY(isTTY) + { + state_.lock()->active = isTTY; + updateThread = std::thread([&]() { + auto state(state_.lock()); + while (state->active) { + if (!state->haveUpdate) + state.wait(updateCV); + draw(*state); + state.wait_for(quitCV, std::chrono::milliseconds(50)); + } + }); + } + + ~ProgressBar() + { + stop(); + updateThread.join(); + } + + void stop() + { + auto state(state_.lock()); + if (!state->active) return; + state->active = false; + std::string status = getStatus(*state); + writeToStderr("\r\e[K"); + if (status != "") + writeToStderr("[" + status + "]\n"); + updateCV.notify_one(); + quitCV.notify_one(); + } + + void log(Verbosity lvl, const FormatOrString & fs) override + { + auto state(state_.lock()); + log(*state, lvl, fs.s); + } + + void log(State & state, Verbosity lvl, const std::string & s) + { + if (state.active) { + writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); + draw(state); + } else { + auto s2 = s + ANSI_NORMAL "\n"; + if (!isTTY) s2 = filterANSIEscapes(s2, true); + writeToStderr(s2); + } + } + + void startActivity(ActivityId act, Verbosity lvl, ActivityType type, + const std::string & s, const Fields & fields, ActivityId parent) override + { + auto state(state_.lock()); + + if (lvl <= verbosity && !s.empty()) + log(*state, lvl, s + "..."); + + state->activities.emplace_back(ActInfo()); + auto i = std::prev(state->activities.end()); + i->s = s; + i->type = type; + i->parent = parent; + state->its.emplace(act, i); + state->activitiesByType[type].its.emplace(act, i); + + if (type == actBuild) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name); + auto machineName = getS(fields, 1); + if (machineName != "") + i->s += fmt(" on " ANSI_BOLD "%s" ANSI_NORMAL, machineName); + auto curRound = getI(fields, 2); + auto nrRounds = getI(fields, 3); + if (nrRounds != 1) + i->s += fmt(" (round %d/%d)", curRound, nrRounds); + i->name = DrvName(name).name; + } + + if (type == actSubstitute) { + auto name = storePathToName(getS(fields, 0)); + auto sub = getS(fields, 1); + i->s = fmt( + hasPrefix(sub, "local") + ? "copying " ANSI_BOLD "%s" ANSI_NORMAL " from %s" + : "fetching " ANSI_BOLD "%s" ANSI_NORMAL " from %s", + name, sub); + } + + if (type == actPostBuildHook) { + auto name = storePathToName(getS(fields, 0)); + if (hasSuffix(name, ".drv")) + name.resize(name.size() - 4); + i->s = fmt("post-build " ANSI_BOLD "%s" ANSI_NORMAL, name); + i->name = DrvName(name).name; + } + + if (type == actQueryPathInfo) { + auto name = storePathToName(getS(fields, 0)); + i->s = fmt("querying " ANSI_BOLD "%s" ANSI_NORMAL " on %s", name, getS(fields, 1)); + } + + if ((type == actDownload && hasAncestor(*state, actCopyPath, parent)) + || (type == actDownload && hasAncestor(*state, actQueryPathInfo, parent)) + || (type == actCopyPath && hasAncestor(*state, actSubstitute, parent))) + i->visible = false; + + update(*state); + } + + /* Check whether an activity has an ancestore with the specified + type. */ + bool hasAncestor(State & state, ActivityType type, ActivityId act) + { + while (act != 0) { + auto i = state.its.find(act); + if (i == state.its.end()) break; + if (i->second->type == type) return true; + act = i->second->parent; + } + return false; + } + + void stopActivity(ActivityId act) override + { + auto state(state_.lock()); + + auto i = state->its.find(act); + if (i != state->its.end()) { + + auto & actByType = state->activitiesByType[i->second->type]; + actByType.done += i->second->done; + actByType.failed += i->second->failed; + + for (auto & j : i->second->expectedByType) + state->activitiesByType[j.first].expected -= j.second; + + actByType.its.erase(act); + state->activities.erase(i->second); + state->its.erase(i); + } + + update(*state); + } + + void result(ActivityId act, ResultType type, const std::vector & fields) override + { + auto state(state_.lock()); + + if (type == resFileLinked) { + state->filesLinked++; + state->bytesLinked += getI(fields, 0); + update(*state); + } + + else if (type == resBuildLogLine || type == resPostBuildLogLine) { + auto lastLine = trim(getS(fields, 0)); + if (!lastLine.empty()) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo info = *i->second; + if (printBuildLogs) { + auto suffix = "> "; + if (type == resPostBuildLogLine) { + suffix = " (post)> "; + } + log(*state, lvlInfo, ANSI_FAINT + info.name.value_or("unnamed") + suffix + ANSI_NORMAL + lastLine); + } else { + state->activities.erase(i->second); + info.lastLine = lastLine; + state->activities.emplace_back(info); + i->second = std::prev(state->activities.end()); + update(*state); + } + } + } + + else if (type == resUntrustedPath) { + state->untrustedPaths++; + update(*state); + } + + else if (type == resCorruptedPath) { + state->corruptedPaths++; + update(*state); + } + + else if (type == resSetPhase) { + auto i = state->its.find(act); + assert(i != state->its.end()); + i->second->phase = getS(fields, 0); + update(*state); + } + + else if (type == resProgress) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.done = getI(fields, 0); + actInfo.expected = getI(fields, 1); + actInfo.running = getI(fields, 2); + actInfo.failed = getI(fields, 3); + update(*state); + } + + else if (type == resSetExpected) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + auto type = (ActivityType) getI(fields, 0); + auto & j = actInfo.expectedByType[type]; + state->activitiesByType[type].expected -= j; + j = getI(fields, 1); + state->activitiesByType[type].expected += j; + update(*state); + } + } + + void update(State & state) + { + state.haveUpdate = true; + updateCV.notify_one(); + } + + void draw(State & state) + { + state.haveUpdate = false; + if (!state.active) return; + + std::string line; + + std::string status = getStatus(state); + if (!status.empty()) { + line += '['; + line += status; + line += "]"; + } + + if (!state.activities.empty()) { + if (!status.empty()) line += " "; + auto i = state.activities.rbegin(); + + while (i != state.activities.rend() && (!i->visible || (i->s.empty() && i->lastLine.empty()))) + ++i; + + if (i != state.activities.rend()) { + line += i->s; + if (!i->phase.empty()) { + line += " ("; + line += i->phase; + line += ")"; + } + if (!i->lastLine.empty()) { + if (!i->s.empty()) line += ": "; + line += i->lastLine; + } + } + } + + auto width = getWindowSize().second; + if (width <= 0) width = std::numeric_limits::max(); + + writeToStderr("\r" + filterANSIEscapes(line, false, width) + "\e[K"); + } + + std::string getStatus(State & state) + { + auto MiB = 1024.0 * 1024.0; + + std::string res; + + auto renderActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { + auto & act = state.activitiesByType[type]; + uint64_t done = act.done, expected = act.done, running = 0, failed = act.failed; + for (auto & j : act.its) { + done += j.second->done; + expected += j.second->expected; + running += j.second->running; + failed += j.second->failed; + } + + expected = std::max(expected, act.expected); + + std::string s; + + if (running || done || expected || failed) { + if (running) + if (expected != 0) + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + running / unit, done / unit, expected / unit); + else + s = fmt(ANSI_BLUE + numberFmt + ANSI_NORMAL "/" ANSI_GREEN + numberFmt + ANSI_NORMAL, + running / unit, done / unit); + else if (expected != done) + if (expected != 0) + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL "/" + numberFmt, + done / unit, expected / unit); + else + s = fmt(ANSI_GREEN + numberFmt + ANSI_NORMAL, done / unit); + else + s = fmt(done ? ANSI_GREEN + numberFmt + ANSI_NORMAL : numberFmt, done / unit); + s = fmt(itemFmt, s); + + if (failed) + s += fmt(" (" ANSI_RED "%d failed" ANSI_NORMAL ")", failed / unit); + } + + return s; + }; + + auto showActivity = [&](ActivityType type, const std::string & itemFmt, const std::string & numberFmt = "%d", double unit = 1) { + auto s = renderActivity(type, itemFmt, numberFmt, unit); + if (s.empty()) return; + if (!res.empty()) res += ", "; + res += s; + }; + + showActivity(actBuilds, "%s built"); + + auto s1 = renderActivity(actCopyPaths, "%s copied"); + auto s2 = renderActivity(actCopyPath, "%s MiB", "%.1f", MiB); + + if (!s1.empty() || !s2.empty()) { + if (!res.empty()) res += ", "; + if (s1.empty()) res += "0 copied"; else res += s1; + if (!s2.empty()) { res += " ("; res += s2; res += ')'; } + } + + showActivity(actDownload, "%s MiB DL", "%.1f", MiB); + + { + auto s = renderActivity(actOptimiseStore, "%s paths optimised"); + if (s != "") { + s += fmt(", %.1f MiB / %d inodes freed", state.bytesLinked / MiB, state.filesLinked); + if (!res.empty()) res += ", "; + res += s; + } + } + + // FIXME: don't show "done" paths in green. + showActivity(actVerifyPaths, "%s paths verified"); + + if (state.corruptedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d corrupted" ANSI_NORMAL, state.corruptedPaths); + } + + if (state.untrustedPaths) { + if (!res.empty()) res += ", "; + res += fmt(ANSI_RED "%d untrusted" ANSI_NORMAL, state.untrustedPaths); + } + + return res; + } +}; + +void startProgressBar(bool printBuildLogs) +{ + logger = new ProgressBar( + printBuildLogs, + isatty(STDERR_FILENO) && getEnv("TERM", "dumb") != "dumb"); +} + +void stopProgressBar() +{ + auto progressBar = dynamic_cast(logger); + if (progressBar) progressBar->stop(); + +} + +} diff --git a/third_party/nix/src/nix/progress-bar.hh b/third_party/nix/src/nix/progress-bar.hh new file mode 100644 index 000000000..4d61175c2 --- /dev/null +++ b/third_party/nix/src/nix/progress-bar.hh @@ -0,0 +1,11 @@ +#pragma once + +#include "logging.hh" + +namespace nix { + +void startProgressBar(bool printBuildLogs = false); + +void stopProgressBar(); + +} diff --git a/third_party/nix/src/nix/repl.cc b/third_party/nix/src/nix/repl.cc new file mode 100644 index 000000000..f857b2e89 --- /dev/null +++ b/third_party/nix/src/nix/repl.cc @@ -0,0 +1,791 @@ +#include +#include +#include +#include + +#include + +#ifdef READLINE +#include +#include +#else +// editline < 1.15.2 don't wrap their API for C++ usage +// (added in https://github.com/troglobit/editline/commit/91398ceb3427b730995357e9d120539fb9bb7461). +// This results in linker errors due to to name-mangling of editline C symbols. +// For compatibility with these versions, we wrap the API here +// (wrapping multiple times on newer versions is no problem). +extern "C" { +#include +} +#endif + +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "common-eval-args.hh" +#include "get-drvs.hh" +#include "derivations.hh" +#include "affinity.hh" +#include "globals.hh" +#include "command.hh" +#include "finally.hh" + +namespace nix { + +#define ESC_RED "\033[31m" +#define ESC_GRE "\033[32m" +#define ESC_YEL "\033[33m" +#define ESC_BLU "\033[34;1m" +#define ESC_MAG "\033[35m" +#define ESC_CYA "\033[36m" +#define ESC_END "\033[0m" + +struct NixRepl +{ + string curDir; + EvalState state; + Bindings * autoArgs; + + Strings loadedFiles; + + const static int envSize = 32768; + StaticEnv staticEnv; + Env * env; + int displ; + StringSet varNames; + + const Path historyFile; + + NixRepl(const Strings & searchPath, nix::ref store); + ~NixRepl(); + void mainLoop(const std::vector & files); + StringSet completePrefix(string prefix); + bool getLine(string & input, const std::string &prompt); + Path getDerivationPath(Value & v); + bool processLine(string line); + void loadFile(const Path & path); + void initEnv(); + void reloadFiles(); + void addAttrsToScope(Value & attrs); + void addVarToScope(const Symbol & name, Value & v); + Expr * parseString(string s); + void evalString(string s, Value & v); + + typedef set ValuesSeen; + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth); + std::ostream & printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen); +}; + + +void printHelp() +{ + std::cout + << "Usage: nix-repl [--help] [--version] [-I path] paths...\n" + << "\n" + << "nix-repl is a simple read-eval-print loop (REPL) for the Nix package manager.\n" + << "\n" + << "Options:\n" + << " --help\n" + << " Prints out a summary of the command syntax and exits.\n" + << "\n" + << " --version\n" + << " Prints out the Nix version number on standard output and exits.\n" + << "\n" + << " -I path\n" + << " Add a path to the Nix expression search path. This option may be given\n" + << " multiple times. See the NIX_PATH environment variable for information on\n" + << " the semantics of the Nix search path. Paths added through -I take\n" + << " precedence over NIX_PATH.\n" + << "\n" + << " paths...\n" + << " A list of paths to files containing Nix expressions which nix-repl will\n" + << " load and add to its scope.\n" + << "\n" + << " A path surrounded in < and > will be looked up in the Nix expression search\n" + << " path, as in the Nix language itself.\n" + << "\n" + << " If an element of paths starts with http:// or https://, it is interpreted\n" + << " as the URL of a tarball that will be downloaded and unpacked to a temporary\n" + << " location. The tarball must include a single top-level directory containing\n" + << " at least a file named default.nix.\n"; +} + + +string removeWhitespace(string s) +{ + s = chomp(s); + size_t n = s.find_first_not_of(" \n\r\t"); + if (n != string::npos) s = string(s, n); + return s; +} + + +NixRepl::NixRepl(const Strings & searchPath, nix::ref store) + : state(searchPath, store) + , staticEnv(false, &state.staticBaseEnv) + , historyFile(getDataDir() + "/nix/repl-history") +{ + curDir = absPath("."); +} + + +NixRepl::~NixRepl() +{ + write_history(historyFile.c_str()); +} + +static NixRepl * curRepl; // ugly + +static char * completionCallback(char * s, int *match) { + auto possible = curRepl->completePrefix(s); + if (possible.size() == 1) { + *match = 1; + auto *res = strdup(possible.begin()->c_str() + strlen(s)); + if (!res) throw Error("allocation failure"); + return res; + } else if (possible.size() > 1) { + auto checkAllHaveSameAt = [&](size_t pos) { + auto &first = *possible.begin(); + for (auto &p : possible) { + if (p.size() <= pos || p[pos] != first[pos]) + return false; + } + return true; + }; + size_t start = strlen(s); + size_t len = 0; + while (checkAllHaveSameAt(start + len)) ++len; + if (len > 0) { + *match = 1; + auto *res = strdup(std::string(*possible.begin(), start, len).c_str()); + if (!res) throw Error("allocation failure"); + return res; + } + } + + *match = 0; + return nullptr; +} + +static int listPossibleCallback(char *s, char ***avp) { + auto possible = curRepl->completePrefix(s); + + if (possible.size() > (INT_MAX / sizeof(char*))) + throw Error("too many completions"); + + int ac = 0; + char **vp = nullptr; + + auto check = [&](auto *p) { + if (!p) { + if (vp) { + while (--ac >= 0) + free(vp[ac]); + free(vp); + } + throw Error("allocation failure"); + } + return p; + }; + + vp = check((char **)malloc(possible.size() * sizeof(char*))); + + for (auto & p : possible) + vp[ac++] = check(strdup(p.c_str())); + + *avp = vp; + + return ac; +} + +namespace { + // Used to communicate to NixRepl::getLine whether a signal occurred in ::readline. + volatile sig_atomic_t g_signal_received = 0; + + void sigintHandler(int signo) { + g_signal_received = signo; + } +} + +void NixRepl::mainLoop(const std::vector & files) +{ + string error = ANSI_RED "error:" ANSI_NORMAL " "; + std::cout << "Welcome to Nix version " << nixVersion << ". Type :? for help." << std::endl << std::endl; + + for (auto & i : files) + loadedFiles.push_back(i); + + reloadFiles(); + if (!loadedFiles.empty()) std::cout << std::endl; + + // Allow nix-repl specific settings in .inputrc + rl_readline_name = "nix-repl"; + createDirs(dirOf(historyFile)); +#ifndef READLINE + el_hist_size = 1000; +#endif + read_history(historyFile.c_str()); + curRepl = this; +#ifndef READLINE + rl_set_complete_func(completionCallback); + rl_set_list_possib_func(listPossibleCallback); +#endif + + std::string input; + + while (true) { + // When continuing input from previous lines, don't print a prompt, just align to the same + // number of chars as the prompt. + if (!getLine(input, input.empty() ? "nix-repl> " : " ")) + break; + + try { + if (!removeWhitespace(input).empty() && !processLine(input)) return; + } catch (ParseError & e) { + if (e.msg().find("unexpected $end") != std::string::npos) { + // For parse errors on incomplete input, we continue waiting for the next line of + // input without clearing the input so far. + continue; + } else { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } + } catch (Error & e) { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } catch (Interrupted & e) { + printMsg(lvlError, format(error + "%1%%2%") % (settings.showTrace ? e.prefix() : "") % e.msg()); + } + + // We handled the current input fully, so we should clear it + // and read brand new input. + input.clear(); + std::cout << std::endl; + } +} + + +bool NixRepl::getLine(string & input, const std::string &prompt) +{ + struct sigaction act, old; + sigset_t savedSignalMask, set; + + auto setupSignals = [&]() { + act.sa_handler = sigintHandler; + sigfillset(&act.sa_mask); + act.sa_flags = 0; + if (sigaction(SIGINT, &act, &old)) + throw SysError("installing handler for SIGINT"); + + sigemptyset(&set); + sigaddset(&set, SIGINT); + if (sigprocmask(SIG_UNBLOCK, &set, &savedSignalMask)) + throw SysError("unblocking SIGINT"); + }; + auto restoreSignals = [&]() { + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); + + if (sigaction(SIGINT, &old, 0)) + throw SysError("restoring handler for SIGINT"); + }; + + setupSignals(); + char * s = readline(prompt.c_str()); + Finally doFree([&]() { free(s); }); + restoreSignals(); + + if (g_signal_received) { + g_signal_received = 0; + input.clear(); + return true; + } + + if (!s) + return false; + input += s; + input += '\n'; + return true; +} + + +StringSet NixRepl::completePrefix(string prefix) +{ + StringSet completions; + + size_t start = prefix.find_last_of(" \n\r\t(){}[]"); + std::string prev, cur; + if (start == std::string::npos) { + prev = ""; + cur = prefix; + } else { + prev = std::string(prefix, 0, start + 1); + cur = std::string(prefix, start + 1); + } + + size_t slash, dot; + + if ((slash = cur.rfind('/')) != string::npos) { + try { + auto dir = std::string(cur, 0, slash); + auto prefix2 = std::string(cur, slash + 1); + for (auto & entry : readDirectory(dir == "" ? "/" : dir)) { + if (entry.name[0] != '.' && hasPrefix(entry.name, prefix2)) + completions.insert(prev + dir + "/" + entry.name); + } + } catch (Error &) { + } + } else if ((dot = cur.rfind('.')) == string::npos) { + /* This is a variable name; look it up in the current scope. */ + StringSet::iterator i = varNames.lower_bound(cur); + while (i != varNames.end()) { + if (string(*i, 0, cur.size()) != cur) break; + completions.insert(prev + *i); + i++; + } + } else { + try { + /* This is an expression that should evaluate to an + attribute set. Evaluate it to get the names of the + attributes. */ + string expr(cur, 0, dot); + string cur2 = string(cur, dot + 1); + + Expr * e = parseString(expr); + Value v; + e->eval(state, *env, v); + state.forceAttrs(v); + + for (auto & i : *v.attrs) { + string name = i.name; + if (string(name, 0, cur2.size()) != cur2) continue; + completions.insert(prev + expr + "." + name); + } + + } catch (ParseError & e) { + // Quietly ignore parse errors. + } catch (EvalError & e) { + // Quietly ignore evaluation errors. + } catch (UndefinedVarError & e) { + // Quietly ignore undefined variable errors. + } + } + + return completions; +} + + +static int runProgram(const string & program, const Strings & args) +{ + Strings args2(args); + args2.push_front(program); + + Pid pid; + pid = fork(); + if (pid == -1) throw SysError("forking"); + if (pid == 0) { + restoreAffinity(); + execvp(program.c_str(), stringsToCharPtrs(args2).data()); + _exit(1); + } + + return pid.wait(); +} + + +bool isVarName(const string & s) +{ + if (s.size() == 0) return false; + char c = s[0]; + if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; + for (auto & i : s) + if (!((i >= 'a' && i <= 'z') || + (i >= 'A' && i <= 'Z') || + (i >= '0' && i <= '9') || + i == '_' || i == '-' || i == '\'')) + return false; + return true; +} + + +Path NixRepl::getDerivationPath(Value & v) { + auto drvInfo = getDerivation(state, v, false); + if (!drvInfo) + throw Error("expression does not evaluate to a derivation, so I can't build it"); + Path drvPath = drvInfo->queryDrvPath(); + if (drvPath == "" || !state.store->isValidPath(drvPath)) + throw Error("expression did not evaluate to a valid derivation"); + return drvPath; +} + + +bool NixRepl::processLine(string line) +{ + if (line == "") return true; + + string command, arg; + + if (line[0] == ':') { + size_t p = line.find_first_of(" \n\r\t"); + command = string(line, 0, p); + if (p != string::npos) arg = removeWhitespace(string(line, p)); + } else { + arg = line; + } + + if (command == ":?" || command == ":help") { + std::cout + << "The following commands are available:\n" + << "\n" + << " Evaluate and print expression\n" + << " = Bind expression to variable\n" + << " :a Add attributes from resulting set to scope\n" + << " :b Build derivation\n" + << " :i Build derivation, then install result into current profile\n" + << " :l Load Nix expression and add it to scope\n" + << " :p Evaluate and print expression recursively\n" + << " :q Exit nix-repl\n" + << " :r Reload all files\n" + << " :s Build dependencies of derivation, then start nix-shell\n" + << " :t Describe result of evaluation\n" + << " :u Build derivation, then start nix-shell\n"; + } + + else if (command == ":a" || command == ":add") { + Value v; + evalString(arg, v); + addAttrsToScope(v); + } + + else if (command == ":l" || command == ":load") { + state.resetFileCache(); + loadFile(arg); + } + + else if (command == ":r" || command == ":reload") { + state.resetFileCache(); + reloadFiles(); + } + + else if (command == ":t") { + Value v; + evalString(arg, v); + std::cout << showType(v) << std::endl; + + } else if (command == ":u") { + Value v, f, result; + evalString(arg, v); + evalString("drv: (import {}).runCommand \"shell\" { buildInputs = [ drv ]; } \"\"", f); + state.callFunction(f, v, result, Pos()); + + Path drvPath = getDerivationPath(result); + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + + else if (command == ":b" || command == ":i" || command == ":s") { + Value v; + evalString(arg, v); + Path drvPath = getDerivationPath(v); + + if (command == ":b") { + /* We could do the build in this process using buildPaths(), + but doing it in a child makes it easier to recover from + problems / SIGINT. */ + if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPath}) == 0) { + Derivation drv = readDerivation(drvPath); + std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; + for (auto & i : drv.outputs) + std::cout << format(" %1% -> %2%") % i.first % i.second.path << std::endl; + } + } else if (command == ":i") { + runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPath}); + } else { + runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPath}); + } + } + + else if (command == ":p" || command == ":print") { + Value v; + evalString(arg, v); + printValue(std::cout, v, 1000000000) << std::endl; + } + + else if (command == ":q" || command == ":quit") + return false; + + else if (command != "") + throw Error(format("unknown command '%1%'") % command); + + else { + size_t p = line.find('='); + string name; + if (p != string::npos && + p < line.size() && + line[p + 1] != '=' && + isVarName(name = removeWhitespace(string(line, 0, p)))) + { + Expr * e = parseString(string(line, p + 1)); + Value & v(*state.allocValue()); + v.type = tThunk; + v.thunk.env = env; + v.thunk.expr = e; + addVarToScope(state.symbols.create(name), v); + } else { + Value v; + evalString(line, v); + printValue(std::cout, v, 1) << std::endl; + } + } + + return true; +} + + +void NixRepl::loadFile(const Path & path) +{ + loadedFiles.remove(path); + loadedFiles.push_back(path); + Value v, v2; + state.evalFile(lookupFileArg(state, path), v); + state.autoCallFunction(*autoArgs, v, v2); + addAttrsToScope(v2); +} + + +void NixRepl::initEnv() +{ + env = &state.allocEnv(envSize); + env->up = &state.baseEnv; + displ = 0; + staticEnv.vars.clear(); + + varNames.clear(); + for (auto & i : state.staticBaseEnv.vars) + varNames.insert(i.first); +} + + +void NixRepl::reloadFiles() +{ + initEnv(); + + Strings old = loadedFiles; + loadedFiles.clear(); + + bool first = true; + for (auto & i : old) { + if (!first) std::cout << std::endl; + first = false; + std::cout << format("Loading '%1%'...") % i << std::endl; + loadFile(i); + } +} + + +void NixRepl::addAttrsToScope(Value & attrs) +{ + state.forceAttrs(attrs); + for (auto & i : *attrs.attrs) + addVarToScope(i.name, *i.value); + std::cout << format("Added %1% variables.") % attrs.attrs->size() << std::endl; +} + + +void NixRepl::addVarToScope(const Symbol & name, Value & v) +{ + if (displ >= envSize) + throw Error("environment full; cannot add more variables"); + staticEnv.vars[name] = displ; + env->values[displ++] = &v; + varNames.insert((string) name); +} + + +Expr * NixRepl::parseString(string s) +{ + Expr * e = state.parseExprFromString(s, curDir, staticEnv); + return e; +} + + +void NixRepl::evalString(string s, Value & v) +{ + Expr * e = parseString(s); + e->eval(state, *env, v); + state.forceValue(v); +} + + +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth) +{ + ValuesSeen seen; + return printValue(str, v, maxDepth, seen); +} + + +std::ostream & printStringValue(std::ostream & str, const char * string) { + str << "\""; + for (const char * i = string; *i; i++) + if (*i == '\"' || *i == '\\') str << "\\" << *i; + else if (*i == '\n') str << "\\n"; + else if (*i == '\r') str << "\\r"; + else if (*i == '\t') str << "\\t"; + else str << *i; + str << "\""; + return str; +} + + +// FIXME: lot of cut&paste from Nix's eval.cc. +std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int maxDepth, ValuesSeen & seen) +{ + str.flush(); + checkInterrupt(); + + state.forceValue(v); + + switch (v.type) { + + case tInt: + str << ESC_CYA << v.integer << ESC_END; + break; + + case tBool: + str << ESC_CYA << (v.boolean ? "true" : "false") << ESC_END; + break; + + case tString: + str << ESC_YEL; + printStringValue(str, v.string.s); + str << ESC_END; + break; + + case tPath: + str << ESC_GRE << v.path << ESC_END; // !!! escaping? + break; + + case tNull: + str << ESC_CYA "null" ESC_END; + break; + + case tAttrs: { + seen.insert(&v); + + bool isDrv = state.isDerivation(v); + + if (isDrv) { + str << "«derivation "; + Bindings::iterator i = v.attrs->find(state.sDrvPath); + PathSet context; + Path drvPath = i != v.attrs->end() ? state.coerceToPath(*i->pos, *i->value, context) : "???"; + str << drvPath << "»"; + } + + else if (maxDepth > 0) { + str << "{ "; + + typedef std::map Sorted; + Sorted sorted; + for (auto & i : *v.attrs) + sorted[i.name] = i.value; + + for (auto & i : sorted) { + if (isVarName(i.first)) + str << i.first; + else + printStringValue(str, i.first.c_str()); + str << " = "; + if (seen.find(i.second) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *i.second, maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << "; "; + } + + str << "}"; + } else + str << "{ ... }"; + + break; + } + + case tList1: + case tList2: + case tListN: + seen.insert(&v); + + str << "[ "; + if (maxDepth > 0) + for (unsigned int n = 0; n < v.listSize(); ++n) { + if (seen.find(v.listElems()[n]) != seen.end()) + str << "«repeated»"; + else + try { + printValue(str, *v.listElems()[n], maxDepth - 1, seen); + } catch (AssertionError & e) { + str << ESC_RED "«error: " << e.msg() << "»" ESC_END; + } + str << " "; + } + else + str << "... "; + str << "]"; + break; + + case tLambda: { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ESC_BLU "«lambda @ " << filterANSIEscapes(s.str()) << "»" ESC_END; + break; + } + + case tPrimOp: + str << ESC_MAG "«primop»" ESC_END; + break; + + case tPrimOpApp: + str << ESC_BLU "«primop-app»" ESC_END; + break; + + case tFloat: + str << v.fpoint; + break; + + default: + str << ESC_RED "«unknown»" ESC_END; + break; + } + + return str; +} + +struct CmdRepl : StoreCommand, MixEvalArgs +{ + std::vector files; + + CmdRepl() + { + expectArgs("files", &files); + } + + std::string name() override { return "repl"; } + + std::string description() override + { + return "start an interactive environment for evaluating Nix expressions"; + } + + void run(ref store) override + { + auto repl = std::make_unique(searchPath, openStore()); + repl->autoArgs = getAutoArgs(repl->state); + repl->mainLoop(files); + } +}; + +static RegisterCommand r1(make_ref()); + +} diff --git a/third_party/nix/src/nix/run.cc b/third_party/nix/src/nix/run.cc new file mode 100644 index 000000000..90b76d666 --- /dev/null +++ b/third_party/nix/src/nix/run.cc @@ -0,0 +1,260 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "local-store.hh" +#include "finally.hh" +#include "fs-accessor.hh" +#include "progress-bar.hh" +#include "affinity.hh" + +#if __linux__ +#include +#endif + +#include + +using namespace nix; + +std::string chrootHelperName = "__run_in_chroot"; + +struct CmdRun : InstallablesCommand +{ + std::vector command = { "bash" }; + StringSet keep, unset; + bool ignoreEnvironment = false; + + CmdRun() + { + mkFlag() + .longName("command") + .shortName('c') + .description("command and arguments to be executed; defaults to 'bash'") + .labels({"command", "args"}) + .arity(ArityAny) + .handler([&](std::vector ss) { + if (ss.empty()) throw UsageError("--command requires at least one argument"); + command = ss; + }); + + mkFlag() + .longName("ignore-environment") + .shortName('i') + .description("clear the entire environment (except those specified with --keep)") + .set(&ignoreEnvironment, true); + + mkFlag() + .longName("keep") + .shortName('k') + .description("keep specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector ss) { keep.insert(ss.front()); }); + + mkFlag() + .longName("unset") + .shortName('u') + .description("unset specified environment variable") + .arity(1) + .labels({"name"}) + .handler([&](std::vector ss) { unset.insert(ss.front()); }); + } + + std::string name() override + { + return "run"; + } + + std::string description() override + { + return "run a shell in which the specified packages are available"; + } + + Examples examples() override + { + return { + Example{ + "To start a shell providing GNU Hello from NixOS 17.03:", + "nix run -f channel:nixos-17.03 hello" + }, + Example{ + "To start a shell providing youtube-dl from your 'nixpkgs' channel:", + "nix run nixpkgs.youtube-dl" + }, + Example{ + "To run GNU Hello:", + "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" + }, + Example{ + "To run GNU Hello in a chroot store:", + "nix run --store ~/my-nix nixpkgs.hello -c hello" + }, + }; + } + + void run(ref store) override + { + auto outPaths = toStorePaths(store, Build, installables); + + auto accessor = store->getFSAccessor(); + + if (ignoreEnvironment) { + + if (!unset.empty()) + throw UsageError("--unset does not make sense with --ignore-environment"); + + std::map kept; + for (auto & var : keep) { + auto s = getenv(var.c_str()); + if (s) kept[var] = s; + } + + clearEnv(); + + for (auto & var : kept) + setenv(var.first.c_str(), var.second.c_str(), 1); + + } else { + + if (!keep.empty()) + throw UsageError("--keep does not make sense without --ignore-environment"); + + for (auto & var : unset) + unsetenv(var.c_str()); + } + + std::unordered_set done; + std::queue todo; + for (auto & path : outPaths) todo.push(path); + + auto unixPath = tokenizeString(getEnv("PATH"), ":"); + + while (!todo.empty()) { + Path path = todo.front(); + todo.pop(); + if (!done.insert(path).second) continue; + + if (true) + unixPath.push_front(path + "/bin"); + + auto propPath = path + "/nix-support/propagated-user-env-packages"; + if (accessor->stat(propPath).type == FSAccessor::tRegular) { + for (auto & p : tokenizeString(readFile(propPath))) + todo.push(p); + } + } + + setenv("PATH", concatStringsSep(":", unixPath).c_str(), 1); + + std::string cmd = *command.begin(); + Strings args; + for (auto & arg : command) args.push_back(arg); + + stopProgressBar(); + + restoreSignals(); + + restoreAffinity(); + + /* If this is a diverted store (i.e. its "logical" location + (typically /nix/store) differs from its "physical" location + (e.g. /home/eelco/nix/store), then run the command in a + chroot. For non-root users, this requires running it in new + mount and user namespaces. Unfortunately, + unshare(CLONE_NEWUSER) doesn't work in a multithreaded + program (which "nix" is), so we exec() a single-threaded + helper program (chrootHelper() below) to do the work. */ + auto store2 = store.dynamic_pointer_cast(); + + if (store2 && store->storeDir != store2->realStoreDir) { + Strings helperArgs = { chrootHelperName, store->storeDir, store2->realStoreDir, cmd }; + for (auto & arg : args) helperArgs.push_back(arg); + + execv(readLink("/proc/self/exe").c_str(), stringsToCharPtrs(helperArgs).data()); + + throw SysError("could not execute chroot helper"); + } + + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to exec '%s'", cmd); + } +}; + +static RegisterCommand r1(make_ref()); + +void chrootHelper(int argc, char * * argv) +{ + int p = 1; + std::string storeDir = argv[p++]; + std::string realStoreDir = argv[p++]; + std::string cmd = argv[p++]; + Strings args; + while (p < argc) + args.push_back(argv[p++]); + +#if __linux__ + uid_t uid = getuid(); + uid_t gid = getgid(); + + if (unshare(CLONE_NEWUSER | CLONE_NEWNS) == -1) + /* Try with just CLONE_NEWNS in case user namespaces are + specifically disabled. */ + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + + /* Bind-mount realStoreDir on /nix/store. If the latter mount + point doesn't already exists, we have to create a chroot + environment containing the mount point and bind mounts for the + children of /. Would be nice if we could use overlayfs here, + but that doesn't work in a user namespace yet (Ubuntu has a + patch for this: + https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ + if (!pathExists(storeDir)) { + // FIXME: Use overlayfs? + + Path tmpDir = createTempDir(); + + createDirs(tmpDir + storeDir); + + if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + + for (auto entry : readDirectory("/")) { + auto src = "/" + entry.name; + auto st = lstat(src); + if (!S_ISDIR(st.st_mode)) continue; + Path dst = tmpDir + "/" + entry.name; + if (pathExists(dst)) continue; + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); + } + + char * cwd = getcwd(0, 0); + if (!cwd) throw SysError("getting current directory"); + Finally freeCwd([&]() { free(cwd); }); + + if (chroot(tmpDir.c_str()) == -1) + throw SysError(format("chrooting into '%s'") % tmpDir); + + if (chdir(cwd) == -1) + throw SysError(format("chdir to '%s' in chroot") % cwd); + } else + if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + + writeFile("/proc/self/setgroups", "deny"); + writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); + writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + + execvp(cmd.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("unable to exec '%s'", cmd); + +#else + throw Error("mounting the Nix store on '%s' is not supported on this platform", storeDir); +#endif +} diff --git a/third_party/nix/src/nix/search.cc b/third_party/nix/src/nix/search.cc new file mode 100644 index 000000000..eb75493e4 --- /dev/null +++ b/third_party/nix/src/nix/search.cc @@ -0,0 +1,280 @@ +#include "command.hh" +#include "globals.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "names.hh" +#include "get-drvs.hh" +#include "common-args.hh" +#include "json.hh" +#include "json-to-value.hh" +#include "shared.hh" + +#include +#include + +using namespace nix; + +std::string wrap(std::string prefix, std::string s) +{ + return prefix + s + ANSI_NORMAL; +} + +std::string hilite(const std::string & s, const std::smatch & m, std::string postfix) +{ + return + m.empty() + ? s + : std::string(m.prefix()) + + ANSI_RED + std::string(m.str()) + postfix + + std::string(m.suffix()); +} + +struct CmdSearch : SourceExprCommand, MixJSON +{ + std::vector res; + + bool writeCache = true; + bool useCache = true; + + CmdSearch() + { + expectArgs("regex", &res); + + mkFlag() + .longName("update-cache") + .shortName('u') + .description("update the package search cache") + .handler([&]() { writeCache = true; useCache = false; }); + + mkFlag() + .longName("no-cache") + .description("do not use or update the package search cache") + .handler([&]() { writeCache = false; useCache = false; }); + } + + std::string name() override + { + return "search"; + } + + std::string description() override + { + return "query available packages"; + } + + Examples examples() override + { + return { + Example{ + "To show all available packages:", + "nix search" + }, + Example{ + "To show any packages containing 'blender' in its name or description:", + "nix search blender" + }, + Example{ + "To search for Firefox or Chromium:", + "nix search 'firefox|chromium'" + }, + Example{ + "To search for git and frontend or gui:", + "nix search git 'frontend|gui'" + } + }; + } + + void run(ref store) override + { + settings.readOnlyMode = true; + + // Empty search string should match all packages + // Use "^" here instead of ".*" due to differences in resulting highlighting + // (see #1893 -- libc++ claims empty search string is not in POSIX grammar) + if (res.empty()) { + res.push_back("^"); + } + + std::vector regexes; + regexes.reserve(res.size()); + + for (auto &re : res) { + regexes.push_back(std::regex(re, std::regex::extended | std::regex::icase)); + } + + auto state = getEvalState(); + + auto jsonOut = json ? std::make_unique(std::cout) : nullptr; + + auto sToplevel = state->symbols.create("_toplevel"); + auto sRecurse = state->symbols.create("recurseForDerivations"); + + bool fromCache = false; + + std::map results; + + std::function doExpr; + + doExpr = [&](Value * v, std::string attrPath, bool toplevel, JSONObject * cache) { + debug("at attribute '%s'", attrPath); + + try { + uint found = 0; + + state->forceValue(*v); + + if (v->type == tLambda && toplevel) { + Value * v2 = state->allocValue(); + state->autoCallFunction(*state->allocBindings(1), *v, *v2); + v = v2; + state->forceValue(*v); + } + + if (state->isDerivation(*v)) { + + DrvInfo drv(*state, attrPath, v->attrs); + std::string description; + std::smatch attrPathMatch; + std::smatch descriptionMatch; + std::smatch nameMatch; + std::string name; + + DrvName parsed(drv.queryName()); + + for (auto ®ex : regexes) { + std::regex_search(attrPath, attrPathMatch, regex); + + name = parsed.name; + std::regex_search(name, nameMatch, regex); + + description = drv.queryMetaString("description"); + std::replace(description.begin(), description.end(), '\n', ' '); + std::regex_search(description, descriptionMatch, regex); + + if (!attrPathMatch.empty() + || !nameMatch.empty() + || !descriptionMatch.empty()) + { + found++; + } + } + + if (found == res.size()) { + if (json) { + + auto jsonElem = jsonOut->object(attrPath); + + jsonElem.attr("pkgName", parsed.name); + jsonElem.attr("version", parsed.version); + jsonElem.attr("description", description); + + } else { + auto name = hilite(parsed.name, nameMatch, "\e[0;2m") + + std::string(parsed.fullName, parsed.name.length()); + results[attrPath] = fmt( + "* %s (%s)\n %s\n", + wrap("\e[0;1m", hilite(attrPath, attrPathMatch, "\e[0;1m")), + wrap("\e[0;2m", hilite(name, nameMatch, "\e[0;2m")), + hilite(description, descriptionMatch, ANSI_NORMAL)); + } + } + + if (cache) { + cache->attr("type", "derivation"); + cache->attr("name", drv.queryName()); + cache->attr("system", drv.querySystem()); + if (description != "") { + auto meta(cache->object("meta")); + meta.attr("description", description); + } + } + } + + else if (v->type == tAttrs) { + + if (!toplevel) { + auto attrs = v->attrs; + Bindings::iterator j = attrs->find(sRecurse); + if (j == attrs->end() || !state->forceBool(*j->value, *j->pos)) { + debug("skip attribute '%s'", attrPath); + return; + } + } + + bool toplevel2 = false; + if (!fromCache) { + Bindings::iterator j = v->attrs->find(sToplevel); + toplevel2 = j != v->attrs->end() && state->forceBool(*j->value, *j->pos); + } + + for (auto & i : *v->attrs) { + auto cache2 = + cache ? std::make_unique(cache->object(i.name)) : nullptr; + doExpr(i.value, + attrPath == "" ? (std::string) i.name : attrPath + "." + (std::string) i.name, + toplevel2 || fromCache, cache2 ? cache2.get() : nullptr); + } + } + + } catch (AssertionError & e) { + } catch (Error & e) { + if (!toplevel) { + e.addPrefix(fmt("While evaluating the attribute '%s':\n", attrPath)); + throw; + } + } + }; + + Path jsonCacheFileName = getCacheDir() + "/nix/package-search.json"; + + if (useCache && pathExists(jsonCacheFileName)) { + + warn("using cached results; pass '-u' to update the cache"); + + Value vRoot; + parseJSON(*state, readFile(jsonCacheFileName), vRoot); + + fromCache = true; + + doExpr(&vRoot, "", true, nullptr); + } + + else { + createDirs(dirOf(jsonCacheFileName)); + + Path tmpFile = fmt("%s.tmp.%d", jsonCacheFileName, getpid()); + + std::ofstream jsonCacheFile; + + try { + // iostream considered harmful + jsonCacheFile.exceptions(std::ofstream::failbit); + jsonCacheFile.open(tmpFile); + + auto cache = writeCache ? std::make_unique(jsonCacheFile, false) : nullptr; + + doExpr(getSourceExpr(*state), "", true, cache.get()); + + } catch (std::exception &) { + /* Fun fact: catching std::ios::failure does not work + due to C++11 ABI shenanigans. + https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66145 */ + if (!jsonCacheFile) + throw Error("error writing to %s", tmpFile); + throw; + } + + if (writeCache && rename(tmpFile.c_str(), jsonCacheFileName.c_str()) == -1) + throw SysError("cannot rename '%s' to '%s'", tmpFile, jsonCacheFileName); + } + + if (results.size() == 0) + throw Error("no results for the given search term(s)!"); + + RunPager pager; + for (auto el : results) std::cout << el.second << "\n"; + + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/show-config.cc b/third_party/nix/src/nix/show-config.cc new file mode 100644 index 000000000..86638b50d --- /dev/null +++ b/third_party/nix/src/nix/show-config.cc @@ -0,0 +1,40 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "json.hh" + +using namespace nix; + +struct CmdShowConfig : Command, MixJSON +{ + CmdShowConfig() + { + } + + std::string name() override + { + return "show-config"; + } + + std::string description() override + { + return "show the Nix configuration"; + } + + void run() override + { + if (json) { + // FIXME: use appropriate JSON types (bool, ints, etc). + JSONObject jsonObj(std::cout); + globalConfig.toJSON(jsonObj); + } else { + std::map settings; + globalConfig.getSettings(settings); + for (auto & s : settings) + std::cout << s.first + " = " + s.second.value + "\n"; + } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/show-derivation.cc b/third_party/nix/src/nix/show-derivation.cc new file mode 100644 index 000000000..ee94fded3 --- /dev/null +++ b/third_party/nix/src/nix/show-derivation.cc @@ -0,0 +1,119 @@ +// FIXME: integrate this with nix path-info? + +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "archive.hh" +#include "json.hh" +#include "derivations.hh" + +using namespace nix; + +struct CmdShowDerivation : InstallablesCommand +{ + bool recursive = false; + + CmdShowDerivation() + { + mkFlag() + .longName("recursive") + .shortName('r') + .description("include the dependencies of the specified derivations") + .set(&recursive, true); + } + + std::string name() override + { + return "show-derivation"; + } + + std::string description() override + { + return "show the contents of a store derivation"; + } + + Examples examples() override + { + return { + Example{ + "To show the store derivation that results from evaluating the Hello package:", + "nix show-derivation nixpkgs.hello" + }, + Example{ + "To show the full derivation graph (if available) that produced your NixOS system:", + "nix show-derivation -r /run/current-system" + }, + }; + } + + void run(ref store) override + { + auto drvPaths = toDerivations(store, installables, true); + + if (recursive) { + PathSet closure; + store->computeFSClosure(drvPaths, closure); + drvPaths = closure; + } + + { + + JSONObject jsonRoot(std::cout, true); + + for (auto & drvPath : drvPaths) { + if (!isDerivation(drvPath)) continue; + + auto drvObj(jsonRoot.object(drvPath)); + + auto drv = readDerivation(drvPath); + + { + auto outputsObj(drvObj.object("outputs")); + for (auto & output : drv.outputs) { + auto outputObj(outputsObj.object(output.first)); + outputObj.attr("path", output.second.path); + if (output.second.hash != "") { + outputObj.attr("hashAlgo", output.second.hashAlgo); + outputObj.attr("hash", output.second.hash); + } + } + } + + { + auto inputsList(drvObj.list("inputSrcs")); + for (auto & input : drv.inputSrcs) + inputsList.elem(input); + } + + { + auto inputDrvsObj(drvObj.object("inputDrvs")); + for (auto & input : drv.inputDrvs) { + auto inputList(inputDrvsObj.list(input.first)); + for (auto & outputId : input.second) + inputList.elem(outputId); + } + } + + drvObj.attr("platform", drv.platform); + drvObj.attr("builder", drv.builder); + + { + auto argsList(drvObj.list("args")); + for (auto & arg : drv.args) + argsList.elem(arg); + } + + { + auto envObj(drvObj.object("env")); + for (auto & var : drv.env) + envObj.attr(var.first, var.second); + } + } + + } + + std::cout << "\n"; + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/sigs.cc b/third_party/nix/src/nix/sigs.cc new file mode 100644 index 000000000..b1825c412 --- /dev/null +++ b/third_party/nix/src/nix/sigs.cc @@ -0,0 +1,149 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "thread-pool.hh" + +#include + +using namespace nix; + +struct CmdCopySigs : StorePathsCommand +{ + Strings substituterUris; + + CmdCopySigs() + { + mkFlag() + .longName("substituter") + .shortName('s') + .labels({"store-uri"}) + .description("use signatures from specified store") + .arity(1) + .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); + } + + std::string name() override + { + return "copy-sigs"; + } + + std::string description() override + { + return "copy path signatures from substituters (like binary caches)"; + } + + void run(ref store, Paths storePaths) override + { + if (substituterUris.empty()) + throw UsageError("you must specify at least one substituter using '-s'"); + + // FIXME: factor out commonality with MixVerify. + std::vector> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStore(s)); + + ThreadPool pool; + + std::string doneLabel = "done"; + std::atomic added{0}; + + //logger->setExpected(doneLabel, storePaths.size()); + + auto doPath = [&](const Path & storePath) { + //Activity act(*logger, lvlInfo, format("getting signatures for '%s'") % storePath); + + checkInterrupt(); + + auto info = store->queryPathInfo(storePath); + + StringSet newSigs; + + for (auto & store2 : substituters) { + try { + auto info2 = store2->queryPathInfo(storePath); + + /* Don't import signatures that don't match this + binary. */ + if (info->narHash != info2->narHash || + info->narSize != info2->narSize || + info->references != info2->references) + continue; + + for (auto & sig : info2->sigs) + if (!info->sigs.count(sig)) + newSigs.insert(sig); + } catch (InvalidPath &) { + } + } + + if (!newSigs.empty()) { + store->addSignatures(storePath, newSigs); + added += newSigs.size(); + } + + //logger->incProgress(doneLabel); + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + printInfo(format("imported %d signatures") % added); + } +}; + +static RegisterCommand r1(make_ref()); + +struct CmdSignPaths : StorePathsCommand +{ + Path secretKeyFile; + + CmdSignPaths() + { + mkFlag() + .shortName('k') + .longName("key-file") + .label("file") + .description("file containing the secret signing key") + .dest(&secretKeyFile); + } + + std::string name() override + { + return "sign-paths"; + } + + std::string description() override + { + return "sign the specified paths"; + } + + void run(ref store, Paths storePaths) override + { + if (secretKeyFile.empty()) + throw UsageError("you must specify a secret key file using '-k'"); + + SecretKey secretKey(readFile(secretKeyFile)); + + size_t added{0}; + + for (auto & storePath : storePaths) { + auto info = store->queryPathInfo(storePath); + + auto info2(*info); + info2.sigs.clear(); + info2.sign(secretKey); + assert(!info2.sigs.empty()); + + if (!info->sigs.count(*info2.sigs.begin())) { + store->addSignatures(storePath, info2.sigs); + added++; + } + } + + printInfo(format("added %d signatures") % added); + } +}; + +static RegisterCommand r3(make_ref()); diff --git a/third_party/nix/src/nix/upgrade-nix.cc b/third_party/nix/src/nix/upgrade-nix.cc new file mode 100644 index 000000000..35c44a70c --- /dev/null +++ b/third_party/nix/src/nix/upgrade-nix.cc @@ -0,0 +1,160 @@ +#include "command.hh" +#include "common-args.hh" +#include "store-api.hh" +#include "download.hh" +#include "eval.hh" +#include "attr-path.hh" +#include "names.hh" +#include "progress-bar.hh" + +using namespace nix; + +struct CmdUpgradeNix : MixDryRun, StoreCommand +{ + Path profileDir; + std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; + + CmdUpgradeNix() + { + mkFlag() + .longName("profile") + .shortName('p') + .labels({"profile-dir"}) + .description("the Nix profile to upgrade") + .dest(&profileDir); + + mkFlag() + .longName("nix-store-paths-url") + .labels({"url"}) + .description("URL of the file that contains the store paths of the latest Nix release") + .dest(&storePathsUrl); + } + + std::string name() override + { + return "upgrade-nix"; + } + + std::string description() override + { + return "upgrade Nix to the latest stable version"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade Nix to the latest stable version:", + "nix upgrade-nix" + }, + Example{ + "To upgrade Nix in a specific profile:", + "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" + }, + }; + } + + void run(ref store) override + { + evalSettings.pureEval = true; + + if (profileDir == "") + profileDir = getProfileDir(store); + + printInfo("upgrading Nix in profile '%s'", profileDir); + + Path storePath; + { + Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); + storePath = getLatestNix(store); + } + + auto version = DrvName(storePathToName(storePath)).version; + + if (dryRun) { + stopProgressBar(); + printError("would upgrade to version %s", version); + return; + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", storePath)); + store->ensurePath(storePath); + } + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", storePath)); + auto program = storePath + "/bin/nix-env"; + auto s = runProgram(program, false, {"--version"}); + if (s.find("Nix") == std::string::npos) + throw Error("could not verify that '%s' works", program); + } + + stopProgressBar(); + + { + Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", storePath, profileDir)); + runProgram(settings.nixBinDir + "/nix-env", false, + {"--profile", profileDir, "-i", storePath, "--no-sandbox"}); + } + + printError(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); + } + + /* Return the profile in which Nix is installed. */ + Path getProfileDir(ref store) + { + Path where; + + for (auto & dir : tokenizeString(getEnv("PATH"), ":")) + if (pathExists(dir + "/nix-env")) { + where = dir; + break; + } + + if (where == "") + throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + + printInfo("found Nix in '%s'", where); + + if (hasPrefix(where, "/run/current-system")) + throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); + + Path profileDir = dirOf(where); + + // Resolve profile to /nix/var/nix/profiles/ link. + while (canonPath(profileDir).find("/profiles/") == std::string::npos && isLink(profileDir)) + profileDir = readLink(profileDir); + + printInfo("found profile '%s'", profileDir); + + Path userEnv = canonPath(profileDir, true); + + if (baseNameOf(where) != "bin" || + !hasSuffix(userEnv, "user-environment")) + throw Error("directory '%s' does not appear to be part of a Nix profile", where); + + if (!store->isValidPath(userEnv)) + throw Error("directory '%s' is not in the Nix store", userEnv); + + return profileDir; + } + + /* Return the store path of the latest stable Nix. */ + Path getLatestNix(ref store) + { + // FIXME: use nixos.org? + auto req = DownloadRequest(storePathsUrl); + auto res = getDownloader()->download(req); + + auto state = std::make_unique(Strings(), store); + auto v = state->allocValue(); + state->eval(state->parseExprFromString(*res.data, "/no-such-path"), *v); + Bindings & bindings(*state->allocBindings(0)); + auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v); + + return state->forceString(*v2); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/verify.cc b/third_party/nix/src/nix/verify.cc new file mode 100644 index 000000000..8893fded5 --- /dev/null +++ b/third_party/nix/src/nix/verify.cc @@ -0,0 +1,178 @@ +#include "command.hh" +#include "shared.hh" +#include "store-api.hh" +#include "sync.hh" +#include "thread-pool.hh" + +#include + +using namespace nix; + +struct CmdVerify : StorePathsCommand +{ + bool noContents = false; + bool noTrust = false; + Strings substituterUris; + size_t sigsNeeded = 0; + + CmdVerify() + { + mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); + mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag() + .longName("substituter") + .shortName('s') + .labels({"store-uri"}) + .description("use signatures from specified store") + .arity(1) + .handler([&](std::vector ss) { substituterUris.push_back(ss[0]); }); + mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + } + + std::string name() override + { + return "verify"; + } + + std::string description() override + { + return "verify the integrity of store paths"; + } + + Examples examples() override + { + return { + Example{ + "To verify the entire Nix store:", + "nix verify --all" + }, + Example{ + "To check whether each path in the closure of Firefox has at least 2 signatures:", + "nix verify -r -n2 --no-contents $(type -p firefox)" + }, + }; + } + + void run(ref store, Paths storePaths) override + { + std::vector> substituters; + for (auto & s : substituterUris) + substituters.push_back(openStore(s)); + + auto publicKeys = getDefaultPublicKeys(); + + Activity act(*logger, actVerifyPaths); + + std::atomic done{0}; + std::atomic untrusted{0}; + std::atomic corrupted{0}; + std::atomic failed{0}; + std::atomic active{0}; + + auto update = [&]() { + act.progress(done, storePaths.size(), active, failed); + }; + + ThreadPool pool; + + auto doPath = [&](const Path & storePath) { + try { + checkInterrupt(); + + Activity act2(*logger, lvlInfo, actUnknown, fmt("checking '%s'", storePath)); + + MaintainCount> mcActive(active); + update(); + + auto info = store->queryPathInfo(storePath); + + if (!noContents) { + + HashSink sink(info->narHash.type); + store->narFromPath(info->path, sink); + + auto hash = sink.finish(); + + if (hash.first != info->narHash) { + corrupted++; + act2.result(resCorruptedPath, info->path); + printError( + format("path '%s' was modified! expected hash '%s', got '%s'") + % info->path % info->narHash.to_string() % hash.first.to_string()); + } + + } + + if (!noTrust) { + + bool good = false; + + if (info->ultimate && !sigsNeeded) + good = true; + + else { + + StringSet sigsSeen; + size_t actualSigsNeeded = std::max(sigsNeeded, (size_t) 1); + size_t validSigs = 0; + + auto doSigs = [&](StringSet sigs) { + for (auto sig : sigs) { + if (sigsSeen.count(sig)) continue; + sigsSeen.insert(sig); + if (validSigs < ValidPathInfo::maxSigs && info->checkSignature(publicKeys, sig)) + validSigs++; + } + }; + + if (info->isContentAddressed(*store)) validSigs = ValidPathInfo::maxSigs; + + doSigs(info->sigs); + + for (auto & store2 : substituters) { + if (validSigs >= actualSigsNeeded) break; + try { + auto info2 = store2->queryPathInfo(info->path); + if (info2->isContentAddressed(*store)) validSigs = ValidPathInfo::maxSigs; + doSigs(info2->sigs); + } catch (InvalidPath &) { + } catch (Error & e) { + printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + } + } + + if (validSigs >= actualSigsNeeded) + good = true; + } + + if (!good) { + untrusted++; + act2.result(resUntrustedPath, info->path); + printError(format("path '%s' is untrusted") % info->path); + } + + } + + done++; + + } catch (Error & e) { + printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what()); + failed++; + } + + update(); + }; + + for (auto & storePath : storePaths) + pool.enqueue(std::bind(doPath, storePath)); + + pool.process(); + + throw Exit( + (corrupted ? 1 : 0) | + (untrusted ? 2 : 0) | + (failed ? 4 : 0)); + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nix/why-depends.cc b/third_party/nix/src/nix/why-depends.cc new file mode 100644 index 000000000..325a2be0a --- /dev/null +++ b/third_party/nix/src/nix/why-depends.cc @@ -0,0 +1,267 @@ +#include "command.hh" +#include "store-api.hh" +#include "progress-bar.hh" +#include "fs-accessor.hh" +#include "shared.hh" + +#include + +using namespace nix; + +static std::string hilite(const std::string & s, size_t pos, size_t len, + const std::string & colour = ANSI_RED) +{ + return + std::string(s, 0, pos) + + colour + + std::string(s, pos, len) + + ANSI_NORMAL + + std::string(s, pos + len); +} + +static std::string filterPrintable(const std::string & s) +{ + std::string res; + for (char c : s) + res += isprint(c) ? c : '.'; + return res; +} + +struct CmdWhyDepends : SourceExprCommand +{ + std::string _package, _dependency; + bool all = false; + + CmdWhyDepends() + { + expectArg("package", &_package); + expectArg("dependency", &_dependency); + + mkFlag() + .longName("all") + .shortName('a') + .description("show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path") + .set(&all, true); + } + + std::string name() override + { + return "why-depends"; + } + + std::string description() override + { + return "show why a package has another package in its closure"; + } + + Examples examples() override + { + return { + Example{ + "To show one path through the dependency graph leading from Hello to Glibc:", + "nix why-depends nixpkgs.hello nixpkgs.glibc" + }, + Example{ + "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", + "nix why-depends --all nixpkgs.thunderbird nixpkgs.xorg.libX11" + }, + Example{ + "To show why Glibc depends on itself:", + "nix why-depends nixpkgs.glibc nixpkgs.glibc" + }, + }; + } + + void run(ref store) override + { + auto package = parseInstallable(*this, store, _package, false); + auto packagePath = toStorePath(store, Build, package); + auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependencyPath = toStorePath(store, NoBuild, dependency); + auto dependencyPathHash = storePathToHash(dependencyPath); + + PathSet closure; + store->computeFSClosure({packagePath}, closure, false, false); + + if (!closure.count(dependencyPath)) { + printError("'%s' does not depend on '%s'", package->what(), dependency->what()); + return; + } + + stopProgressBar(); // FIXME + + auto accessor = store->getFSAccessor(); + + auto const inf = std::numeric_limits::max(); + + struct Node + { + Path path; + PathSet refs; + PathSet rrefs; + size_t dist = inf; + Node * prev = nullptr; + bool queued = false; + bool visited = false; + }; + + std::map graph; + + for (auto & path : closure) + graph.emplace(path, Node{path, store->queryPathInfo(path)->references}); + + // Transpose the graph. + for (auto & node : graph) + for (auto & ref : node.second.refs) + graph[ref].rrefs.insert(node.first); + + /* Run Dijkstra's shortest path algorithm to get the distance + of every path in the closure to 'dependency'. */ + graph[dependencyPath].dist = 0; + + std::priority_queue queue; + + queue.push(&graph.at(dependencyPath)); + + while (!queue.empty()) { + auto & node = *queue.top(); + queue.pop(); + + for (auto & rref : node.rrefs) { + auto & node2 = graph.at(rref); + auto dist = node.dist + 1; + if (dist < node2.dist) { + node2.dist = dist; + node2.prev = &node; + if (!node2.queued) { + node2.queued = true; + queue.push(&node2); + } + } + + } + } + + /* Print the subgraph of nodes that have 'dependency' in their + closure (i.e., that have a non-infinite distance to + 'dependency'). Print every edge on a path between `package` + and `dependency`. */ + std::function printNode; + + const string treeConn = "╠═══"; + const string treeLast = "╚═══"; + const string treeLine = "║ "; + const string treeNull = " "; + + struct BailOut { }; + + printNode = [&](Node & node, const string & firstPad, const string & tailPad) { + assert(node.dist != inf); + std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", + firstPad, + node.visited ? "\e[38;5;244m" : "", + firstPad != "" ? "=> " : "", + node.path); + + if (node.path == dependencyPath && !all + && packagePath != dependencyPath) + throw BailOut(); + + if (node.visited) return; + node.visited = true; + + /* Sort the references by distance to `dependency` to + ensure that the shortest path is printed first. */ + std::multimap refs; + std::set hashes; + + for (auto & ref : node.refs) { + if (ref == node.path && packagePath != dependencyPath) continue; + auto & node2 = graph.at(ref); + if (node2.dist == inf) continue; + refs.emplace(node2.dist, &node2); + hashes.insert(storePathToHash(node2.path)); + } + + /* For each reference, find the files and symlinks that + contain the reference. */ + std::map hits; + + std::function visitPath; + + visitPath = [&](const Path & p) { + auto st = accessor->stat(p); + + auto p2 = p == node.path ? "/" : std::string(p, node.path.size() + 1); + + auto getColour = [&](const std::string & hash) { + return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; + }; + + if (st.type == FSAccessor::Type::tDirectory) { + auto names = accessor->readDirectory(p); + for (auto & name : names) + visitPath(p + "/" + name); + } + + else if (st.type == FSAccessor::Type::tRegular) { + auto contents = accessor->readFile(p); + + for (auto & hash : hashes) { + auto pos = contents.find(hash); + if (pos != std::string::npos) { + size_t margin = 32; + auto pos2 = pos >= margin ? pos - margin : 0; + hits[hash].emplace_back(fmt("%s: …%s…\n", + p2, + hilite(filterPrintable( + std::string(contents, pos2, pos - pos2 + hash.size() + margin)), + pos - pos2, storePathHashLen, + getColour(hash)))); + } + } + } + + else if (st.type == FSAccessor::Type::tSymlink) { + auto target = accessor->readLink(p); + + for (auto & hash : hashes) { + auto pos = target.find(hash); + if (pos != std::string::npos) + hits[hash].emplace_back(fmt("%s -> %s\n", p2, + hilite(target, pos, storePathHashLen, getColour(hash)))); + } + } + }; + + // FIXME: should use scanForReferences(). + + visitPath(node.path); + + RunPager pager; + for (auto & ref : refs) { + auto hash = storePathToHash(ref.second->path); + + bool last = all ? ref == *refs.rbegin() : true; + + for (auto & hit : hits[hash]) { + bool first = hit == *hits[hash].begin(); + std::cout << tailPad + << (first ? (last ? treeLast : treeConn) : (last ? treeNull : treeLine)) + << hit; + if (!all) break; + } + + printNode(*ref.second, + tailPad + (last ? treeNull : treeLine), + tailPad + (last ? treeNull : treeLine)); + } + }; + + try { + printNode(graph.at(packagePath), "", ""); + } catch (BailOut & ) { } + } +}; + +static RegisterCommand r1(make_ref()); diff --git a/third_party/nix/src/nlohmann/json.hpp b/third_party/nix/src/nlohmann/json.hpp new file mode 100644 index 000000000..c9af0bed3 --- /dev/null +++ b/third_party/nix/src/nlohmann/json.hpp @@ -0,0 +1,20406 @@ +/* + __ _____ _____ _____ + __| | __| | | | JSON for Modern C++ +| | |__ | | | | | | version 3.5.0 +|_____|_____|_____|_|___| https://github.com/nlohmann/json + +Licensed under the MIT License . +SPDX-License-Identifier: MIT +Copyright (c) 2013-2018 Niels Lohmann . + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +*/ + +#ifndef NLOHMANN_JSON_HPP +#define NLOHMANN_JSON_HPP + +#define NLOHMANN_JSON_VERSION_MAJOR 3 +#define NLOHMANN_JSON_VERSION_MINOR 5 +#define NLOHMANN_JSON_VERSION_PATCH 0 + +#include // all_of, find, for_each +#include // assert +#include // and, not, or +#include // nullptr_t, ptrdiff_t, size_t +#include // hash, less +#include // initializer_list +#include // istream, ostream +#include // random_access_iterator_tag +#include // accumulate +#include // string, stoi, to_string +#include // declval, forward, move, pair, swap + +// #include +#ifndef NLOHMANN_JSON_FWD_HPP +#define NLOHMANN_JSON_FWD_HPP + +#include // int64_t, uint64_t +#include // map +#include // allocator +#include // string +#include // vector + +/*! +@brief namespace for Niels Lohmann +@see https://github.com/nlohmann +@since version 1.0.0 +*/ +namespace nlohmann +{ +/*! +@brief default JSONSerializer template argument + +This serializer ignores the template arguments and uses ADL +([argument-dependent lookup](https://en.cppreference.com/w/cpp/language/adl)) +for serialization. +*/ +template +struct adl_serializer; + +template class ObjectType = + std::map, + template class ArrayType = std::vector, + class StringType = std::string, class BooleanType = bool, + class NumberIntegerType = std::int64_t, + class NumberUnsignedType = std::uint64_t, + class NumberFloatType = double, + template class AllocatorType = std::allocator, + template class JSONSerializer = + adl_serializer> +class basic_json; + +/*! +@brief JSON Pointer + +A JSON pointer defines a string syntax for identifying a specific value +within a JSON document. It can be used with functions `at` and +`operator[]`. Furthermore, JSON pointers are the base for JSON patches. + +@sa [RFC 6901](https://tools.ietf.org/html/rfc6901) + +@since version 2.0.0 +*/ +template +class json_pointer; + +/*! +@brief default JSON class + +This type is the default specialization of the @ref basic_json class which +uses the standard template types. + +@since version 1.0.0 +*/ +using json = basic_json<>; +} // namespace nlohmann + +#endif + +// #include + + +// This file contains all internal macro definitions +// You MUST include macro_unscope.hpp at the end of json.hpp to undef all of them + +// exclude unsupported compilers +#if !defined(JSON_SKIP_UNSUPPORTED_COMPILER_CHECK) + #if defined(__clang__) + #if (__clang_major__ * 10000 + __clang_minor__ * 100 + __clang_patchlevel__) < 30400 + #error "unsupported Clang version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #elif defined(__GNUC__) && !(defined(__ICC) || defined(__INTEL_COMPILER)) + #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) < 40800 + #error "unsupported GCC version - see https://github.com/nlohmann/json#supported-compilers" + #endif + #endif +#endif + +// disable float-equal warnings on GCC/clang +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wfloat-equal" +#endif + +// disable documentation warnings on clang +#if defined(__clang__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdocumentation" +#endif + +// allow for portable deprecation warnings +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_DEPRECATED __attribute__((deprecated)) +#elif defined(_MSC_VER) + #define JSON_DEPRECATED __declspec(deprecated) +#else + #define JSON_DEPRECATED +#endif + +// allow to disable exceptions +#if (defined(__cpp_exceptions) || defined(__EXCEPTIONS) || defined(_CPPUNWIND)) && !defined(JSON_NOEXCEPTION) + #define JSON_THROW(exception) throw exception + #define JSON_TRY try + #define JSON_CATCH(exception) catch(exception) + #define JSON_INTERNAL_CATCH(exception) catch(exception) +#else + #define JSON_THROW(exception) std::abort() + #define JSON_TRY if(true) + #define JSON_CATCH(exception) if(false) + #define JSON_INTERNAL_CATCH(exception) if(false) +#endif + +// override exception macros +#if defined(JSON_THROW_USER) + #undef JSON_THROW + #define JSON_THROW JSON_THROW_USER +#endif +#if defined(JSON_TRY_USER) + #undef JSON_TRY + #define JSON_TRY JSON_TRY_USER +#endif +#if defined(JSON_CATCH_USER) + #undef JSON_CATCH + #define JSON_CATCH JSON_CATCH_USER + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_CATCH_USER +#endif +#if defined(JSON_INTERNAL_CATCH_USER) + #undef JSON_INTERNAL_CATCH + #define JSON_INTERNAL_CATCH JSON_INTERNAL_CATCH_USER +#endif + +// manual branch prediction +#if defined(__clang__) || defined(__GNUC__) || defined(__GNUG__) + #define JSON_LIKELY(x) __builtin_expect(!!(x), 1) + #define JSON_UNLIKELY(x) __builtin_expect(!!(x), 0) +#else + #define JSON_LIKELY(x) x + #define JSON_UNLIKELY(x) x +#endif + +// C++ language standard detection +#if (defined(__cplusplus) && __cplusplus >= 201703L) || (defined(_HAS_CXX17) && _HAS_CXX17 == 1) // fix for issue #464 + #define JSON_HAS_CPP_17 + #define JSON_HAS_CPP_14 +#elif (defined(__cplusplus) && __cplusplus >= 201402L) || (defined(_HAS_CXX14) && _HAS_CXX14 == 1) + #define JSON_HAS_CPP_14 +#endif + +/*! +@brief macro to briefly define a mapping between an enum and JSON +@def NLOHMANN_JSON_SERIALIZE_ENUM +@since version 3.4.0 +*/ +#define NLOHMANN_JSON_SERIALIZE_ENUM(ENUM_TYPE, ...) \ + template \ + inline void to_json(BasicJsonType& j, const ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [e](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.first == e; \ + }); \ + j = ((it != std::end(m)) ? it : std::begin(m))->second; \ + } \ + template \ + inline void from_json(const BasicJsonType& j, ENUM_TYPE& e) \ + { \ + static_assert(std::is_enum::value, #ENUM_TYPE " must be an enum!"); \ + static const std::pair m[] = __VA_ARGS__; \ + auto it = std::find_if(std::begin(m), std::end(m), \ + [j](const std::pair& ej_pair) -> bool \ + { \ + return ej_pair.second == j; \ + }); \ + e = ((it != std::end(m)) ? it : std::begin(m))->first; \ + } + +// Ugly macros to avoid uglier copy-paste when specializing basic_json. They +// may be removed in the future once the class is split. + +#define NLOHMANN_BASIC_JSON_TPL_DECLARATION \ + template class ObjectType, \ + template class ArrayType, \ + class StringType, class BooleanType, class NumberIntegerType, \ + class NumberUnsignedType, class NumberFloatType, \ + template class AllocatorType, \ + template class JSONSerializer> + +#define NLOHMANN_BASIC_JSON_TPL \ + basic_json + +// #include + + +#include // not +#include // size_t +#include // conditional, enable_if, false_type, integral_constant, is_constructible, is_integral, is_same, remove_cv, remove_reference, true_type + +namespace nlohmann +{ +namespace detail +{ +// alias templates to reduce boilerplate +template +using enable_if_t = typename std::enable_if::type; + +template +using uncvref_t = typename std::remove_cv::type>::type; + +// implementation of C++14 index_sequence and affiliates +// source: https://stackoverflow.com/a/32223343 +template +struct index_sequence +{ + using type = index_sequence; + using value_type = std::size_t; + static constexpr std::size_t size() noexcept + { + return sizeof...(Ints); + } +}; + +template +struct merge_and_renumber; + +template +struct merge_and_renumber, index_sequence> + : index_sequence < I1..., (sizeof...(I1) + I2)... > {}; + +template +struct make_index_sequence + : merge_and_renumber < typename make_index_sequence < N / 2 >::type, + typename make_index_sequence < N - N / 2 >::type > {}; + +template<> struct make_index_sequence<0> : index_sequence<> {}; +template<> struct make_index_sequence<1> : index_sequence<0> {}; + +template +using index_sequence_for = make_index_sequence; + +// dispatch utility (taken from ranges-v3) +template struct priority_tag : priority_tag < N - 1 > {}; +template<> struct priority_tag<0> {}; + +// taken from ranges-v3 +template +struct static_const +{ + static constexpr T value{}; +}; + +template +constexpr T static_const::value; +} // namespace detail +} // namespace nlohmann + +// #include + + +#include // not +#include // numeric_limits +#include // false_type, is_constructible, is_integral, is_same, true_type +#include // declval + +// #include + +// #include + + +#include // random_access_iterator_tag + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template struct make_void +{ + using type = void; +}; +template using void_t = typename make_void::type; +} // namespace detail +} // namespace nlohmann + +// #include + + +namespace nlohmann +{ +namespace detail +{ +template +struct iterator_types {}; + +template +struct iterator_types < + It, + void_t> +{ + using difference_type = typename It::difference_type; + using value_type = typename It::value_type; + using pointer = typename It::pointer; + using reference = typename It::reference; + using iterator_category = typename It::iterator_category; +}; + +// This is required as some compilers implement std::iterator_traits in a way that +// doesn't work with SFINAE. See https://github.com/nlohmann/json/issues/1341. +template +struct iterator_traits +{ +}; + +template +struct iterator_traits < T, enable_if_t < !std::is_pointer::value >> + : iterator_types +{ +}; + +template +struct iterator_traits::value>> +{ + using iterator_category = std::random_access_iterator_tag; + using value_type = T; + using difference_type = ptrdiff_t; + using pointer = T*; + using reference = T&; +}; +} +} + +// #include + +// #include + + +#include + +// #include + + +// http://en.cppreference.com/w/cpp/experimental/is_detected +namespace nlohmann +{ +namespace detail +{ +struct nonesuch +{ + nonesuch() = delete; + ~nonesuch() = delete; + nonesuch(nonesuch const&) = delete; + void operator=(nonesuch const&) = delete; +}; + +template class Op, + class... Args> +struct detector +{ + using value_t = std::false_type; + using type = Default; +}; + +template class Op, class... Args> +struct detector>, Op, Args...> +{ + using value_t = std::true_type; + using type = Op; +}; + +template