refactor(tvix/nar-bridge): have Export return root node

… and nar size / sha256 digest.

Instead of producing sparse PathInfo messages when NARs are sent to
nar-bridge, the nar-bridge http server now keeps a lookup table
(narsha256) -> (rootNode, narSize)

This removes a whole bunch of noise, because we don't need to keep
sparse fields around.

A convenience function
`GenPathInfo(rootNode *castorev1pb.Node, narInfo *narinfo.NarInfo)` is
added, which is used to produce PathInfo messages, either when receiving
a NAR file over http and uploading it to a remote PathInfoService, or to
synthesize the PathInfoMessage to return to the client, if nar-bridge is
acting as a PathInfoService for a remove Nix HTTP Binary cache.

Change-Id: Ibba1ab6238a050816c4fab29cb21ae88877d8613
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9651
Tested-by: BuildkiteCI
Reviewed-by: Brian McGee <brian@bmcgee.ie>

tvix/nar-bridge/pkg/http/narinfo_put.go

@@ -2,14 +2,11 @@ package http
 
 import (
 	"net/http"
-	"path"
 
-	castorev1pb "code.tvl.fyi/tvix/castore/protos"
-	storev1pb "code.tvl.fyi/tvix/store/protos"
+	"code.tvl.fyi/tvix/nar-bridge/pkg/importer"
 	"github.com/go-chi/chi/v5"
 	"github.com/nix-community/go-nix/pkg/narinfo"
 	"github.com/nix-community/go-nix/pkg/nixbase32"
-	"github.com/nix-community/go-nix/pkg/storepath"
 	"github.com/sirupsen/logrus"
 	log "github.com/sirupsen/logrus"
 )
@@ -43,12 +40,10 @@ func registerNarinfoPut(s *Server) {
 			"output_path": narInfo.StorePath,
 		})
 
-		var pathInfo *storev1pb.PathInfo
-
 		// look up the narHash in our temporary map
-		s.narHashToPathInfoMu.Lock()
-		pathInfo, found := s.narHashToPathInfo[narInfo.NarHash.SRIString()]
-		s.narHashToPathInfoMu.Unlock()
+		s.narDbMu.Lock()
+		narData, found := s.narDb[narInfo.NarHash.SRIString()]
+		s.narDbMu.Unlock()
 		if !found {
 			log.Error("unable to find referred NAR")
 			w.WriteHeader(http.StatusBadRequest)
@@ -60,10 +55,12 @@ func registerNarinfoPut(s *Server) {
 			return
 		}
 
+		rootNode := narData.rootNode
+
 		// compare fields with what we computed while receiving the NAR file
 
 		// NarSize needs to match
-		if pathInfo.Narinfo.NarSize != narInfo.NarSize {
+		if narData.narSize != narInfo.NarSize {
 			log.Error("narsize mismatch")
 			w.WriteHeader(http.StatusBadRequest)
 			_, err := w.Write([]byte("unable to parse narinfo"))
@@ -73,90 +70,23 @@ func registerNarinfoPut(s *Server) {
 
 			return
 		}
-		// We know the narhash in the .narinfo matches one of the two narhashes in the partial pathInfo,
-		// because that's how we found it.
 
-		// FUTUREWORK: We can't compare References yet, but it'd be a good idea to
-		// do reference checking on .nar files server-side during upload.
-		// We however still need to be parse them, because we store
-		// the bytes in pathInfo.References, and the full strings in pathInfo.Narinfo.ReferenceNames.
-		referencesBytes := make([][]byte, 0)
-		for _, reference := range narInfo.References {
-			storePath, err := storepath.FromString(reference)
+		pathInfo, err := importer.GenPathInfo(rootNode, narInfo)
+		if err != nil {
+			log.WithError(err).Error("unable to generate PathInfo")
+
+			w.WriteHeader(http.StatusInternalServerError)
+			_, err := w.Write([]byte("unable to generate PathInfo"))
 			if err != nil {
-				log.WithField("reference", reference).WithError(err).Error("unable to parse reference")
-				w.WriteHeader(http.StatusBadRequest)
-				_, err := w.Write([]byte("unable to parse reference"))
-				if err != nil {
-					log.WithError(err).Errorf("unable to write error message to client")
-				}
-
-				return
+				log.WithError(err).Errorf("unable to write error message to client")
 			}
-			referencesBytes = append(referencesBytes, storePath.Digest)
+
+			return
 		}
 
-		// assemble the []*storev1pb.NARInfo_Signature{} from narinfo.Signatures.
-		pbNarinfoSignatures := make([]*storev1pb.NARInfo_Signature, 0)
-		for _, narinfoSig := range narInfo.Signatures {
+		log.WithField("pathInfo", pathInfo).Debug("inserted new pathInfo")
 
-			pbNarinfoSignatures = append(pbNarinfoSignatures, &storev1pb.NARInfo_Signature{
-				Name: narinfoSig.Name,
-				Data: narinfoSig.Data,
-			})
-		}
-
-		// If everything matches, We will add References, NAR signatures and the
-		// output path name, and then upload to the pathinfo service.
-		// We want a copy here, because we don't want to mutate the contents in the lookup table
-		// until we get things back from the remote store.
-		pathInfoToUpload := &storev1pb.PathInfo{
-			Node:       nil, // set below
-			References: referencesBytes,
-			Narinfo: &storev1pb.NARInfo{
-				NarSize:        pathInfo.Narinfo.NarSize,
-				NarSha256:      pathInfo.Narinfo.NarSha256,
-				Signatures:     pbNarinfoSignatures,
-				ReferenceNames: narInfo.References,
-			},
-		}
-
-		// We need to add the basename of the storepath from the .narinfo
-		// to the pathInfo to be sent.
-		switch v := (pathInfo.GetNode().GetNode()).(type) {
-		case *castorev1pb.Node_File:
-			pathInfoToUpload.Node = &castorev1pb.Node{
-				Node: &castorev1pb.Node_File{
-					File: &castorev1pb.FileNode{
-						Name:       []byte(path.Base(narInfo.StorePath)),
-						Digest:     v.File.Digest,
-						Size:       v.File.Size,
-						Executable: v.File.Executable,
-					},
-				},
-			}
-		case *castorev1pb.Node_Symlink:
-			pathInfoToUpload.Node = &castorev1pb.Node{
-				Node: &castorev1pb.Node_Symlink{
-					Symlink: &castorev1pb.SymlinkNode{
-						Name:   []byte(path.Base(narInfo.StorePath)),
-						Target: v.Symlink.Target,
-					},
-				},
-			}
-		case *castorev1pb.Node_Directory:
-			pathInfoToUpload.Node = &castorev1pb.Node{
-				Node: &castorev1pb.Node_Directory{
-					Directory: &castorev1pb.DirectoryNode{
-						Name:   []byte(path.Base(narInfo.StorePath)),
-						Digest: v.Directory.Digest,
-						Size:   v.Directory.Size,
-					},
-				},
-			}
-		}
-
-		receivedPathInfo, err := s.pathInfoServiceClient.Put(ctx, pathInfoToUpload)
+		receivedPathInfo, err := s.pathInfoServiceClient.Put(ctx, pathInfo)
 		if err != nil {
 			log.WithError(err).Error("unable to upload pathinfo to service")
 			w.WriteHeader(http.StatusInternalServerError)
@@ -168,8 +98,6 @@ func registerNarinfoPut(s *Server) {
 			return
 		}
 
-		log.Debugf("received new pathInfo: %v+", receivedPathInfo)
-
-		// TODO: update the local temporary pathinfo with this?
+		log.WithField("pathInfo", receivedPathInfo).Debug("got back PathInfo")
 	})
 }