Rework goals to fix NixOS#11928

First step, dones *NOT* fix the issue, but does introduce some new goal
types.

Fixes NixOS#11928

Author: Ericson2314

src/libstore/build/build-trace-goal.cc

@@ -0,0 +1,258 @@
+#include "nix/store/build/build-trace-goal.hh"
+#include "nix/store/build/derivation-resolution-goal.hh"
+#include "nix/store/build/substitution-goal.hh"
+#include "nix/store/build/worker.hh"
+#include "nix/store/derivations.hh"
+#include "nix/store/store-open.hh"
+#include "nix/util/callback.hh"
+#include "nix/util/finally.hh"
+#include "nix/util/util.hh"
+
+namespace nix {
+
+BuildTraceGoal::BuildTraceGoal(const SingleDerivedPath::Built & id, Worker & worker)
+    : Goal{worker, init()}
+    , id{id}
+{
+    name = fmt("substitution of '%s'", id.to_string(worker.store));
+    trace("created");
+}
+
+Goal::Co BuildTraceGoal::init()
+{
+    trace("init");
+
+    DrvOutput id2{
+        .drvHash = Hash::dummy,
+        .outputName = id.output,
+    };
+
+    StorePath drvPath = StorePath::dummy;
+
+    // No `std::visit` with coroutines :(
+    if (const auto * path = std::get_if<SingleDerivedPath::Opaque>(&*id.drvPath)) {
+        // At least we know the drv path statically, can procede
+        drvPath = path->path;
+    } else if (const auto * outputDeriving = std::get_if<SingleDerivedPath::Built>(&*id.drvPath)) {
+        // Dynamic derivation case, need to resolve that first.
+
+        auto g = worker.makeBuildTraceGoal({
+            outputDeriving->drvPath,
+            outputDeriving->output,
+        });
+
+        co_await await(Goals{upcast_goal(g)});
+
+        if (nrFailed > 0) {
+            co_return amDone(
+                nrNoSubstituters > 0  ? ecNoSubstituters : ecFailed,
+                Error("The output deriving path '%s' could not be resolved", outputDeriving->to_string(worker.store)));
+        }
+
+        drvPath = g->outputInfo->outPath;
+    }
+
+    {
+        auto getStaticHash = [&](Store & store, const Derivation & drv) {
+            // TODO slow
+            auto hashes = staticOutputHashes(store, drv);
+            auto * p = get(hashes, id.output);
+            if (!p)
+                throw Error("Missing output '%s' for derivation '%s'", id.output, worker.store.printStorePath(drvPath));
+            id2.drvHash = *p;
+        };
+
+        if (worker.evalStore.isValidPath(drvPath)) {
+            auto drv = worker.evalStore.readDerivation(drvPath);
+            getStaticHash(worker.evalStore, drv);
+        } else if (worker.store.isValidPath(drvPath)) {
+            auto drv = worker.store.readDerivation(drvPath);
+            getStaticHash(worker.evalStore, drv);
+        }
+    }
+
+    /* If the derivation already exists, we’re done */
+    if ((outputInfo = worker.store.queryRealisation(id2))) {
+        co_return amDone(ecSuccess);
+    }
+
+    /**
+     * Firstly, whether we know the status, secondly, what it is
+     */
+    std::optional<bool> drvIsResolved;
+
+    auto checkStaticOutputs = [&](Store & store) -> std::optional<Goal::Done> {
+        auto drv = store.readDerivation(drvPath);
+        auto os = drv.outputsAndOptPaths(worker.store);
+        /* Mark what we now know */
+        drvIsResolved = {drv.inputDrvs.map.empty()};
+        if (auto * p = get(os, id2.outputName)) {
+            if (auto & outPath = p->second) {
+                outputInfo = std::make_shared<Realisation>(id2, *outPath);
+                return {amDone(ecSuccess)};
+            } else {
+                /* Otherwise, not failure, just looking up build trace below. */
+                return std::nullopt;
+            }
+        } else {
+            return {amDone(
+                ecFailed,
+                Error(
+                    "Derivation '%s' does not have output '%s', impossible to find build trace key-value pair",
+                    worker.store.printStorePath(drvPath),
+                    id2.outputName))};
+        }
+    };
+
+    /* If we have the derivaiton, and the derivation has statically-known output paths */
+    if (worker.evalStore.isValidPath(drvPath)) {
+        if (auto co = checkStaticOutputs(worker.evalStore))
+            co_return *co;
+    } else if (worker.store.isValidPath(drvPath)) {
+        if (auto co = checkStaticOutputs(worker.store))
+            co_return *co;
+    }
+
+    auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+
+    bool substituterFailed = false;
+
+    if (!drvIsResolved || *drvIsResolved) {
+        /* Since derivation might be resolved --- isn't known to be
+           not-resolved, it might have entries. So, let's try querying
+           the substituters. */
+        for (const auto & sub : subs) {
+            trace("trying next substituter");
+
+            /* The callback of the curl download below can outlive `this` (if
+               some other error occurs), so it must not touch `this`. So put
+               the shared state in a separate refcounted object. */
+            auto outPipe = std::make_shared<MuxablePipe>();
+#ifndef _WIN32
+            outPipe->create();
+#else
+            outPipe->createAsyncPipe(worker.ioport.get());
+#endif
+
+            auto promise = std::make_shared<std::promise<std::shared_ptr<const Realisation>>>();
+
+            sub->queryRealisation(
+                id2, {[outPipe(outPipe), promise(promise)](std::future<std::shared_ptr<const Realisation>> res) {
+                    try {
+                        Finally updateStats([&]() { outPipe->writeSide.close(); });
+                        promise->set_value(res.get());
+                    } catch (...) {
+                        promise->set_exception(std::current_exception());
+                    }
+                }});
+
+            worker.childStarted(
+                shared_from_this(),
+                {
+#ifndef _WIN32
+                    outPipe->readSide.get()
+#else
+                    &*outPipe
+#endif
+                },
+                true,
+                false);
+
+            co_await Suspend{};
+
+            worker.childTerminated(this);
+
+            std::shared_ptr<const Realisation> outputInfo;
+            try {
+                outputInfo = promise->get_future().get();
+            } catch (std::exception & e) {
+                printError(e.what());
+                substituterFailed = true;
+            }
+
+            if (!outputInfo)
+                continue;
+
+            worker.store.registerDrvOutput(*outputInfo);
+
+            trace("finished");
+            co_return amDone(ecSuccess);
+        }
+    }
+
+    /* Derivation might not be resolved, let's try doing that */
+    trace("trying resolving derivation in build-trace goal");
+
+    auto g = worker.makeDerivationResolutionGoal(drvPath);
+    g->preserveException = true;
+    co_await await(Goals{g});
+
+    if (nrFailed > 0) {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+
+        if (substituterFailed) {
+            worker.failedSubstitutions++;
+            worker.updateProgress();
+        }
+
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters, g->ex);
+    }
+
+    /* This should be set if the goal succeeded */
+    assert(g->resolvedDrv);
+
+    if (g->resolvedDrvPath != drvPath) {
+        /* Try everything again, now with a resolved derivation */
+        auto bt2 = worker.makeBuildTraceGoal({
+            makeConstantStorePathRef(g->resolvedDrvPath),
+            id2.outputName,
+        });
+
+        /* No longer need 'g' */
+        g = nullptr;
+
+        co_await await(Goals{bt2});
+
+        /* Set the build trace value as our own. Note the signure will not
+           match our key since we're the unresolved derivation, but that's
+           fine. We're not writing it to the DB; that's `bt2`' job. */
+        if (bt2->outputInfo)
+            outputInfo = bt2->outputInfo;
+
+        co_return amDone(bt2->exitCode, bt2->ex);
+    } else {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+        debug("build trace is not known for '%s', derivation is already resolved", id2.to_string());
+
+        if (substituterFailed) {
+            worker.failedSubstitutions++;
+            worker.updateProgress();
+        }
+
+        /* Hack: don't indicate failure if there were no substituters.
+           In that case the calling derivation should just do a
+           build. */
+        co_return amDone(
+            substituterFailed ? ecFailed : ecNoSubstituters,
+            Error("build trace is not known for '%s', derivation is already resolved", id2.to_string()));
+    }
+}
+
+std::string BuildTraceGoal::key()
+{
+    /* "a$" ensures substitution goals happen before derivation
+       goals. */
+    return "a$" + std::string(id.to_string(worker.store));
+}
+
+void BuildTraceGoal::handleEOF(Descriptor fd)
+{
+    worker.wakeUp(shared_from_this());
+}
+
+}

src/libstore/build/derivation-goal.cc

@@ -50,7 +50,7 @@ std::string DerivationGoal::key()
        i.e. a derivation named "aardvark" always comes before
        "baboon". And substitution goals always happen before
        derivation goals (due to "b$"). */
-    return "b$" + std::string(drvPath.name()) + "$" + SingleDerivedPath::Built{
+    return "c$" + std::string(drvPath.name()) + "$" + SingleDerivedPath::Built{
         .drvPath = makeConstantStorePathRef(drvPath),
         .output = wantedOutput,
     }.to_string(worker.store);
@@ -170,6 +170,7 @@ Goal::Co DerivationGoal::haveDerivation()
                 waitees.insert(
                     upcast_goal(
                         worker.makeDrvOutputSubstitutionGoal(
+                            drvPath,
                             DrvOutput{status.outputHash, outputName},
                             buildMode == bmRepair ? Repair : NoRepair
                         )

src/libstore/build/derivation-resolution-goal.cc

@@ -0,0 +1,109 @@
+#include "nix/store/build/derivation-resolution-goal.hh"
+#include "nix/store/build/build-trace-goal.hh"
+#include "nix/util/finally.hh"
+#include "nix/store/build/worker.hh"
+#include "nix/store/build/substitution-goal.hh"
+#include "nix/util/callback.hh"
+#include "nix/store/derivations.hh"
+
+namespace nix {
+
+DerivationResolutionGoal::DerivationResolutionGoal(const StorePath & drvPath, Worker & worker)
+    : Goal(worker, init())
+    , drvPath(drvPath)
+{
+    name = fmt("resolution of '%s'", worker.store.printStorePath(drvPath));
+    trace("created");
+}
+
+Goal::Co DerivationResolutionGoal::init()
+{
+    trace("init");
+
+    std::unique_ptr<Derivation> drv;
+
+    if (worker.evalStore.isValidPath(drvPath)) {
+        drv = std::make_unique<Derivation>(worker.evalStore.readDerivation(drvPath));
+    } else if (worker.store.isValidPath(drvPath)) {
+        drv = std::make_unique<Derivation>(worker.store.readDerivation(drvPath));
+    } else {
+        auto goal0 = worker.makePathSubstitutionGoal(drvPath);
+        goal0->preserveException = true;
+        co_await await(Goals{goal0});
+        if (nrFailed > 0)
+            co_return amDone(goal0->exitCode, goal0->ex);
+
+        drv = std::make_unique<Derivation>(worker.store.readDerivation(drvPath));
+    }
+
+    trace("output path substituted");
+
+    std::map<SingleDerivedPath::Built, std::shared_ptr<BuildTraceGoal>> traceGoals;
+
+    {
+        Goals waitees;
+
+        std::function<void(ref<const SingleDerivedPath>, const DerivedPathMap<StringSet>::ChildNode &)> accumInputPaths;
+
+        accumInputPaths = [&](ref<const SingleDerivedPath> depDrvPath,
+                              const DerivedPathMap<StringSet>::ChildNode & inputNode) {
+            for (auto & outputName : inputNode.value) {
+                SingleDerivedPath::Built key{depDrvPath, outputName};
+                auto goal = worker.makeBuildTraceGoal(key);
+                waitees.insert(goal);
+                traceGoals.insert_or_assign(std::move(key), std::move(goal));
+            }
+
+            for (auto & [outputName, childNode] : inputNode.childMap)
+                accumInputPaths(
+                    make_ref<const SingleDerivedPath>(SingleDerivedPath::Built{depDrvPath, outputName}), childNode);
+        };
+
+        for (auto & [depDrvPath, depNode] : drv->inputDrvs.map)
+            accumInputPaths(makeConstantStorePathRef(depDrvPath), depNode);
+
+        co_await await(std::move(waitees));
+    }
+
+    if (nrFailed > 0) {
+        co_return amDone(
+            nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed,
+            Error("Could not find traces for all inputs for derivation '%s'", worker.store.printStorePath(drvPath)));
+    }
+
+    trace("derivation inputs resolved");
+
+    if (auto d = drv->tryResolve(
+            worker.store,
+            [&](ref<const SingleDerivedPath> depDrvPath, const std::string & outputName) -> std::optional<StorePath> {
+                auto mEntry = get(traceGoals, SingleDerivedPath::Built{depDrvPath, outputName});
+                assert(mEntry);
+                assert(*mEntry);
+                auto outputInfo = (*mEntry)->outputInfo;
+                assert(outputInfo);
+                return outputInfo->outPath;
+            })) {
+        resolvedDrv = std::make_shared<BasicDerivation>(std::move(*d));
+        resolvedDrvPath = writeDerivation(worker.store, Derivation{*resolvedDrv}, NoRepair, true);
+
+        trace("finished");
+        co_return amDone(ecSuccess);
+    } else {
+        /* If we succesfully resolved all build trace goals, `tryResolve` should succeed. */
+        assert(false);
+    }
+}
+
+std::string DerivationResolutionGoal::key()
+{
+    /* "a$" ensures substitution goals happen before derivation
+       goals. */
+    return "b$" + worker.store.printStorePath(drvPath);
+}
+
+void DerivationResolutionGoal::handleEOF(Descriptor fd)
+{
+    worker.wakeUp(shared_from_this());
+}
+
+}