fix(tvix/eval): implement cppnix JSON-serialisation semantics

This drops the usage of serde::Serialize, as the trait can not be used
to implement the correct semantics (function colouring!).

Instead, a manual JSON serialisation function is written which
correctly handles toString, outPath and other similar weirdnesses.

Unexpectedly, the eval-okay-tojson test from the C++ Nix test suite
now passes, too.

This fixes an issue where serialising data structures containing
derivations to JSON would fail.

Change-Id: I5c39e3d8356ee93a07eda481410f88610f6dd9f8
Reviewed-on: https://cl.tvl.fyi/c/depot/+/8209
Reviewed-by: raitobezarius <tvl@lahfa.xyz>
Tested-by: BuildkiteCI

tvix/eval/src/vm/generators.rs

@@ -114,6 +114,10 @@ pub enum GeneratorRequest {
     /// Request evaluation of `builtins.tryEval` from the VM. See
     /// [`VM::catch_result`] for an explanation of how this works.
     TryForce(Value),
+
+    /// Request serialisation of a value to JSON, according to the
+    /// slightly odd Nix evaluation rules.
+    ToJson(Value),
 }
 
 /// Human-readable representation of a generator message, used by observers.
@@ -160,6 +164,7 @@ impl Display for GeneratorRequest {
             GeneratorRequest::ReadDir(p) => write!(f, "read_dir({})", p.to_string_lossy()),
             GeneratorRequest::Span => write!(f, "span"),
             GeneratorRequest::TryForce(v) => write!(f, "try_force({})", v.type_of()),
+            GeneratorRequest::ToJson(v) => write!(f, "to_json({})", v.type_of()),
         }
     }
 }
@@ -440,6 +445,14 @@ impl<'o> VM<'o> {
                             self.enqueue_generator("force", span, |co| value.force(co));
                             return Ok(false);
                         }
+
+                        GeneratorRequest::ToJson(value) => {
+                            self.reenqueue_generator(name, span.clone(), generator);
+                            self.enqueue_generator("to_json", span, |co| {
+                                value.to_json_generator(co)
+                            });
+                            return Ok(false);
+                        }
                     }
                 }
 
@@ -708,6 +721,16 @@ pub(crate) async fn request_span(co: &GenCo) -> LightSpan {
     }
 }
 
+pub(crate) async fn request_to_json(co: &GenCo, value: Value) -> serde_json::Value {
+    match co.yield_(GeneratorRequest::ToJson(value)).await {
+        GeneratorResponse::Value(Value::Json(json)) => json,
+        msg => panic!(
+            "Tvix bug: VM responded with incorrect generator message: {}",
+            msg
+        ),
+    }
+}
+
 /// Call the given value as if it was an attribute set containing a functor. The
 /// arguments must already be prepared on the stack when a generator frame from
 /// this function is invoked.