Improve progress indicator

src/nix/installables.cc

@@ -223,9 +223,9 @@ PathSet InstallablesCommand::buildInstallables(ref<Store> store, bool dryRun)
         buildables.insert(b.begin(), b.end());
     }
 
-    printMissing(store, buildables);
-
-    if (!dryRun)
+    if (dryRun)
+        printMissing(store, buildables);
+    else
         store->buildPaths(buildables);
 
     PathSet outPaths;

src/nix/main.cc

@@ -27,6 +27,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
 
 void mainWrapped(int argc, char * * argv)
 {
+    verbosity = lvlError;
     settings.verboseBuild = false;
 
     initNix();

src/nix/progress-bar.cc

@@ -1,8 +1,12 @@
 #include "progress-bar.hh"
 #include "util.hh"
 #include "sync.hh"
+#include "store-api.hh"
 
 #include <map>
+#include <atomic>
+
+#include <sys/ioctl.h>
 
 namespace nix {
 
@@ -12,31 +16,47 @@ private:
 
     struct ActInfo
     {
-        Activity * activity;
-        Verbosity lvl;
-        std::string s;
+        std::string s, s2;
     };
 
-    struct Progress
+    struct DownloadInfo
     {
-        uint64_t expected = 0, progress = 0;
+        std::string uri;
+        uint64_t current = 0;
+        uint64_t expected = 0;
     };
 
     struct State
     {
+        std::map<Activity::t, Path> builds;
+        std::set<Activity::t> runningBuilds;
+        uint64_t succeededBuilds = 0;
+        uint64_t failedBuilds = 0;
+        std::map<Activity::t, Path> substitutions;
+        std::set<Activity::t> runningSubstitutions;
+        uint64_t succeededSubstitutions = 0;
+        uint64_t downloadedBytes = 0; // finished downloads
+        std::map<Activity::t, DownloadInfo> downloads;
         std::list<ActInfo> activities;
-        std::map<Activity *, std::list<ActInfo>::iterator> its;
-        std::map<std::string, Progress> progress;
+        std::map<Activity::t, std::list<ActInfo>::iterator> its;
     };
 
     Sync<State> state_;
 
+    int width = 0;
+
 public:
 
+    ProgressBar()
+    {
+        struct winsize ws;
+        if (ioctl(1, TIOCGWINSZ, &ws) == 0)
+            width = ws.ws_col;
+    }
+
     ~ProgressBar()
     {
         auto state(state_.lock());
-        assert(state->activities.empty());
         writeToStderr("\r\e[K");
     }
 
@@ -52,52 +72,36 @@ public:
         update(state);
     }
 
-    void startActivity(Activity & activity, Verbosity lvl, const FormatOrString & fs) override
+    void createActivity(State & state, Activity::t activity, const std::string & s)
     {
-        if (lvl > verbosity) return;
-        auto state(state_.lock());
-        state->activities.emplace_back(ActInfo{&activity, lvl, fs.s});
-        state->its.emplace(&activity, std::prev(state->activities.end()));
-        update(*state);
+        state.activities.emplace_back(ActInfo{s});
+        state.its.emplace(activity, std::prev(state.activities.end()));
     }
 
-    void stopActivity(Activity & activity) override
+    void deleteActivity(State & state, Activity::t activity)
     {
-        auto state(state_.lock());
-        auto i = state->its.find(&activity);
-        if (i == state->its.end()) return;
-        state->activities.erase(i->second);
-        state->its.erase(i);
-        update(*state);
+        auto i = state.its.find(activity);
+        if (i != state.its.end()) {
+            state.activities.erase(i->second);
+            state.its.erase(i);
+        }
     }
 
-    void setExpected(const std::string & label, uint64_t value) override
+    void updateActivity(State & state, Activity::t activity, const std::string & s2)
     {
-        auto state(state_.lock());
-        state->progress[label].expected = value;
-    }
-
-    void setProgress(const std::string & label, uint64_t value) override
-    {
-        auto state(state_.lock());
-        state->progress[label].progress = value;
-    }
-
-    void incExpected(const std::string & label, uint64_t value) override
-    {
-        auto state(state_.lock());
-        state->progress[label].expected += value;
-    }
-
-    void incProgress(const std::string & label, uint64_t value) override
-    {
-        auto state(state_.lock());
-        state->progress[label].progress += value;
+        auto i = state.its.find(activity);
+        assert(i != state.its.end());
+        ActInfo info = *i->second;
+        state.activities.erase(i->second);
+        info.s2 = s2;
+        state.activities.emplace_back(info);
+        i->second = std::prev(state.activities.end());
     }
 
     void update()
     {
         auto state(state_.lock());
+        update(*state);
     }
 
     void update(State & state)
@@ -113,28 +117,169 @@ public:
 
         if (!state.activities.empty()) {
             if (!status.empty()) line += " ";
-            line += state.activities.rbegin()->s;
+            auto i = state.activities.rbegin();
+            line += i->s;
+            if (!i->s2.empty()) {
+                line += ": ";
+                line += i->s2;
+            }
         }
 
         line += "\e[K";
-        writeToStderr(line);
+        writeToStderr(std::string(line, 0, width - 1));
     }
 
     std::string getStatus(State & state)
     {
         std::string res;
-        for (auto & p : state.progress)
-            if (p.second.expected || p.second.progress) {
-                if (!res.empty()) res += ", ";
-                res += std::to_string(p.second.progress);
-                if (p.second.expected) {
-                    res += "/";
-                    res += std::to_string(p.second.expected);
-                }
-                res += " "; res += p.first;
+
+        if (state.failedBuilds) {
+            if (!res.empty()) res += ", ";
+            res += fmt(ANSI_RED "%d failed" ANSI_NORMAL, state.failedBuilds);
+        }
+
+        if (!state.builds.empty() || state.succeededBuilds)
+        {
+            if (!res.empty()) res += ", ";
+            if (!state.runningBuilds.empty())
+                res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningBuilds.size());
+            res += fmt(ANSI_GREEN "%d/%d built" ANSI_NORMAL,
+                state.succeededBuilds, state.succeededBuilds + state.builds.size());
+        }
+
+        if (!state.substitutions.empty() || state.succeededSubstitutions) {
+            if (!res.empty()) res += ", ";
+            if (!state.runningSubstitutions.empty())
+                res += fmt(ANSI_BLUE "%d" "/" ANSI_NORMAL, state.runningSubstitutions.size());
+            res += fmt(ANSI_GREEN "%d/%d fetched" ANSI_NORMAL,
+                state.succeededSubstitutions,
+                state.succeededSubstitutions + state.substitutions.size());
+        }
+
+        if (!state.downloads.empty() || state.downloadedBytes) {
+            if (!res.empty()) res += ", ";
+            uint64_t expected = state.downloadedBytes, current = state.downloadedBytes;
+            for (auto & i : state.downloads) {
+                expected += i.second.expected;
+                current += i.second.current;
             }
+            res += fmt("%1$.0f/%2$.0f KiB", current / 1024.0, expected / 1024.0);
+        }
+
         return res;
     }
+
+    void event(const Event & ev) override
+    {
+        if (ev.type == evBuildCreated) {
+            auto state(state_.lock());
+            state->builds[ev.getI(0)] = ev.getS(1);
+            update(*state);
+        }
+
+        if (ev.type == evBuildStarted) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            state->runningBuilds.insert(act);
+            auto name = storePathToName(state->builds[act]);
+            if (hasSuffix(name, ".drv"))
+                name.resize(name.size() - 4);
+            createActivity(*state, act, fmt("building " ANSI_BOLD "%s" ANSI_NORMAL, name));
+            update(*state);
+        }
+
+        if (ev.type == evBuildFinished) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            if (ev.getI(1)) {
+                if (state->runningBuilds.count(act))
+                    state->succeededBuilds++;
+            } else
+                state->failedBuilds++;
+            state->runningBuilds.erase(act);
+            state->builds.erase(act);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
+        if (ev.type == evBuildOutput) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            assert(state->runningBuilds.count(act));
+            updateActivity(*state, act, ev.getS(1));
+            update(*state);
+        }
+
+        if (ev.type == evSubstitutionCreated) {
+            auto state(state_.lock());
+            state->substitutions[ev.getI(0)] = ev.getS(1);
+            update(*state);
+        }
+
+        if (ev.type == evSubstitutionStarted) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            state->runningSubstitutions.insert(act);
+            auto name = storePathToName(state->substitutions[act]);
+            createActivity(*state, act, fmt("fetching " ANSI_BOLD "%s" ANSI_NORMAL, name));
+            update(*state);
+        }
+
+        if (ev.type == evSubstitutionFinished) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            if (ev.getI(1)) {
+                if (state->runningSubstitutions.count(act))
+                    state->succeededSubstitutions++;
+            }
+            state->runningSubstitutions.erase(act);
+            state->substitutions.erase(act);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
+        if (ev.type == evDownloadCreated) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            std::string uri = ev.getS(1);
+            state->downloads.emplace(act, DownloadInfo{uri});
+            if (state->runningSubstitutions.empty()) // FIXME: hack
+                createActivity(*state, act, fmt("downloading " ANSI_BOLD "%s" ANSI_NORMAL "", uri));
+            update(*state);
+        }
+
+        if (ev.type == evDownloadProgress) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto i = state->downloads.find(act);
+            assert(i != state->downloads.end());
+            i->second.expected = ev.getI(1);
+            i->second.current = ev.getI(2);
+            update(*state);
+        }
+
+        if (ev.type == evDownloadSucceeded) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto i = state->downloads.find(act);
+            assert(i != state->downloads.end());
+            state->downloadedBytes += ev.getI(1);
+            state->downloads.erase(i);
+            deleteActivity(*state, act);
+            update(*state);
+        }
+
+        if (ev.type == evDownloadDestroyed) {
+            auto state(state_.lock());
+            Activity::t act = ev.getI(0);
+            auto i = state->downloads.find(act);
+            if (i != state->downloads.end()) {
+                state->downloads.erase(i);
+                deleteActivity(*state, act);
+                update(*state);
+            }
+        }
+    }
 };
 
 StartProgressBar::StartProgressBar()

src/nix/sigs.cc

@@ -42,10 +42,10 @@ struct CmdCopySigs : StorePathsCommand
         std::string doneLabel = "done";
         std::atomic<size_t> added{0};
 
-        logger->setExpected(doneLabel, storePaths.size());
+        //logger->setExpected(doneLabel, storePaths.size());
 
         auto doPath = [&](const Path & storePath) {
-            Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath);
+            //Activity act(*logger, lvlInfo, format("getting signatures for ‘%s’") % storePath);
 
             checkInterrupt();
 
@@ -76,7 +76,7 @@ struct CmdCopySigs : StorePathsCommand
                 added += newSigs.size();
             }
 
-            logger->incProgress(doneLabel);
+            //logger->incProgress(doneLabel);
         };
 
         for (auto & storePath : storePaths)

src/nix/verify.cc

@@ -65,7 +65,7 @@ struct CmdVerify : StorePathsCommand
         std::string untrustedLabel("untrusted");
         std::string corruptedLabel("corrupted");
         std::string failedLabel("failed");
-        logger->setExpected(doneLabel, storePaths.size());
+        //logger->setExpected(doneLabel, storePaths.size());
 
         ThreadPool pool;
 
@@ -73,7 +73,7 @@ struct CmdVerify : StorePathsCommand
             try {
                 checkInterrupt();
 
-                Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath);
+                //Activity act(*logger, lvlInfo, format("checking ‘%s’") % storePath);
 
                 auto info = store->queryPathInfo(storePath);
 
@@ -85,7 +85,7 @@ struct CmdVerify : StorePathsCommand
                     auto hash = sink.finish();
 
                     if (hash.first != info->narHash) {
-                        logger->incProgress(corruptedLabel);
+                        //logger->incProgress(corruptedLabel);
                         corrupted = 1;
                         printError(
                             format("path ‘%s’ was modified! expected hash ‘%s’, got ‘%s’")
@@ -137,19 +137,19 @@ struct CmdVerify : StorePathsCommand
                     }
 
                     if (!good) {
-                        logger->incProgress(untrustedLabel);
+                        //logger->incProgress(untrustedLabel);
                         untrusted++;
                         printError(format("path ‘%s’ is untrusted") % info->path);
                     }
 
                 }
 
-                logger->incProgress(doneLabel);
+                //logger->incProgress(doneLabel);
                 done++;
 
             } catch (Error & e) {
                 printError(format(ANSI_RED "error:" ANSI_NORMAL " %s") % e.what());
-                logger->incProgress(failedLabel);
+                //logger->incProgress(failedLabel);
                 failed++;
             }
         };