maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(tvix/nar-bridge): have Export return root node

Browse source 
… and nar size / sha256 digest.

Instead of producing sparse PathInfo messages when NARs are sent to
nar-bridge, the nar-bridge http server now keeps a lookup table
(narsha256) -> (rootNode, narSize)

This removes a whole bunch of noise, because we don't need to keep
sparse fields around.

A convenience function
`GenPathInfo(rootNode *castorev1pb.Node, narInfo *narinfo.NarInfo)` is
added, which is used to produce PathInfo messages, either when receiving
a NAR file over http and uploading it to a remote PathInfoService, or to
synthesize the PathInfoMessage to return to the client, if nar-bridge is
acting as a PathInfoService for a remove Nix HTTP Binary cache.

Change-Id: Ibba1ab6238a050816c4fab29cb21ae88877d8613
Reviewed-on: https://cl.tvl.fyi/c/depot/+/9651
Tested-by: BuildkiteCI
Reviewed-by: Brian McGee <brian@bmcgee.ie>
This commit is contained in:
Florian Klink 2023-10-11 12:28:10 +02:00 • committed by flokli
parent ceb1674e9f
commit 98c17147c6
10 changed files with 211 additions and 297 deletions
Download patch file Download diff file Expand all files Collapse all files

18
tvix/nar-bridge/pkg/http/nar_get.go
View file

@ -29,16 +29,18 @@ func renderNar(
log *log.Entry,
directoryServiceClient castorev1pb.DirectoryServiceClient,
blobServiceClient castorev1pb.BlobServiceClient,
narHashToPathInfoMu *sync.Mutex,
narHashToPathInfo map[string]*storev1pb.PathInfo,
narHashDbMu *sync.Mutex,
narHashDb map[string]*narData,
w io.Writer,
narHash *nixhash.Hash,
headOnly bool,
) error {
// look in the lookup table
narHashToPathInfoMu.Lock()
pathInfo, found := narHashToPathInfo[narHash.SRIString()]
narHashToPathInfoMu.Unlock()
narHashDbMu.Lock()
narData, found := narHashDb[narHash.SRIString()]
narHashDbMu.Unlock()
rootNode := narData.rootNode
// if we didn't find anything, return 404.
if !found {
@ -53,7 +55,7 @@ func renderNar(
directories := make(map[string]*castorev1pb.Directory)
// If the root node is a directory, ask the directory service for all directories
if pathInfoDirectory := pathInfo.GetNode().GetDirectory(); pathInfoDirectory != nil {
if pathInfoDirectory := rootNode.GetDirectory(); pathInfoDirectory != nil {
rootDirectoryDigest := pathInfoDirectory.GetDigest()
log = log.WithField("root_directory", base64.StdEncoding.EncodeToString(rootDirectoryDigest))
@ -95,7 +97,7 @@ func renderNar(
// render the NAR file
err := storev1pb.Export(
w,
pathInfo.Node,
rootNode,
func(directoryDigest []byte) (*castorev1pb.Directory, error) {
log.WithField("directory", base64.StdEncoding.EncodeToString(directoryDigest)).Debug("Get directory")
directoryRefStr := hex.EncodeToString(directoryDigest)
@ -177,7 +179,7 @@ func registerNarGet(s *Server) {
log := log.WithField("narhash_url", narHash.SRIString())
// TODO: inline more of that function here?
err = renderNar(ctx, log, s.directoryServiceClient, s.blobServiceClient, &s.narHashToPathInfoMu, s.narHashToPathInfo, w, narHash, isHead)
err = renderNar(ctx, log, s.directoryServiceClient, s.blobServiceClient, &s.narDbMu, s.narDb, w, narHash, isHead)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
w.WriteHeader(http.StatusNotFound)

25
tvix/nar-bridge/pkg/http/nar_put.go
View file

@ -39,7 +39,7 @@ func registerNarPut(s *Server) {
directoriesUploader := importer.NewDirectoriesUploader(ctx, s.directoryServiceClient)
defer directoriesUploader.Done() //nolint:errcheck
pathInfo, err := importer.Import(
rootNode, narSize, narSha256, err := importer.Import(
ctx,
// buffer the body by 10MiB
bufio.NewReaderSize(r.Body, 10*1024*1024),
@ -80,7 +80,7 @@ func registerNarPut(s *Server) {
// This check ensures the server-side came up with the same root hash.
if directoriesPutResponse != nil {
rootDigestPathInfo := pathInfo.GetNode().GetDirectory().GetDigest()
rootDigestPathInfo := rootNode.GetDirectory().GetDigest()
rootDigestDirectoriesPutResponse := directoriesPutResponse.GetRootDigest()
log := log.WithFields(logrus.Fields{
@ -102,17 +102,18 @@ func registerNarPut(s *Server) {
// Compare the nar hash specified in the URL with the one that has been
// calculated while processing the NAR file
piNarHash, err := nixhash.ParseNixBase32(
"sha256:" + nixbase32.EncodeToString(pathInfo.GetNarinfo().NarSha256),
// TODO: bump go-nix and remove the parsing
narHash, err := nixhash.ParseNixBase32(
"sha256:" + nixbase32.EncodeToString(narSha256),
)
if err != nil {
panic("must parse nixbase32")
}
if !bytes.Equal(narHashFromUrl.Digest(), piNarHash.Digest()) {
if !bytes.Equal(narHashFromUrl.Digest(), narHash.Digest()) {
log := log.WithFields(logrus.Fields{
"narhash_received_sha256": piNarHash.SRIString(),
"narsize": pathInfo.GetNarinfo().GetNarSize(),
"narhash_received_sha256": narHash.SRIString(),
"narsize": narSize,
})
log.Error("received bytes don't match narhash from URL")
@ -123,7 +124,6 @@ func registerNarPut(s *Server) {
}
return
}
// Insert the partial pathinfo structs into our lookup map,
@ -131,9 +131,12 @@ func registerNarPut(s *Server) {
// The same might exist already, but it'll have the same contents (so
// replacing will be a no-op), except maybe the root node Name field value, which
// is safe to ignore (as not part of the NAR).
s.narHashToPathInfoMu.Lock()
s.narHashToPathInfo[piNarHash.SRIString()] = pathInfo
s.narHashToPathInfoMu.Unlock()
s.narDbMu.Lock()
s.narDb[narHash.SRIString()] = &narData{
rootNode: rootNode,
narSize: narSize,
}
s.narDbMu.Unlock()
// Done!
})

10
tvix/nar-bridge/pkg/http/narinfo_get.go
View file

@ -29,7 +29,7 @@ func renderNarinfo(
log *log.Entry,
pathInfoServiceClient storev1pb.PathInfoServiceClient,
narHashToPathInfoMu *sync.Mutex,
narHashToPathInfo map[string]*storev1pb.PathInfo,
narHashToPathInfo map[string]*narData,
outputHash []byte,
w io.Writer,
headOnly bool,
@ -51,6 +51,7 @@ func renderNarinfo(
return fmt.Errorf("unable to get pathinfo: %w", err)
}
// TODO: don't parse
narHash, err := nixhash.ParseNixBase32("sha256:" + nixbase32.EncodeToString(pathInfo.GetNarinfo().GetNarSha256()))
if err != nil {
// TODO: return proper error
@ -59,7 +60,10 @@ func renderNarinfo(
// add things to the lookup table, in case the same process didn't handle the NAR hash yet.
narHashToPathInfoMu.Lock()
narHashToPathInfo[narHash.SRIString()] = pathInfo
narHashToPathInfo[narHash.SRIString()] = &narData{
rootNode: pathInfo.GetNode(),
narSize: pathInfo.GetNarinfo().GetNarSize(),
}
narHashToPathInfoMu.Unlock()
if headOnly {
@ -102,7 +106,7 @@ func registerNarinfoGet(s *Server) {
return
}
err = renderNarinfo(ctx, log, s.pathInfoServiceClient, &s.narHashToPathInfoMu, s.narHashToPathInfo, outputHash, w, false)
err = renderNarinfo(ctx, log, s.pathInfoServiceClient, &s.narDbMu, s.narDb, outputHash, w, false)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
w.WriteHeader(http.StatusNotFound)

110
tvix/nar-bridge/pkg/http/narinfo_put.go
View file

@ -2,14 +2,11 @@ package http
import (
"net/http"
"path"
castorev1pb "code.tvl.fyi/tvix/castore/protos"
storev1pb "code.tvl.fyi/tvix/store/protos"
"code.tvl.fyi/tvix/nar-bridge/pkg/importer"
"github.com/go-chi/chi/v5"
"github.com/nix-community/go-nix/pkg/narinfo"
"github.com/nix-community/go-nix/pkg/nixbase32"
"github.com/nix-community/go-nix/pkg/storepath"
"github.com/sirupsen/logrus"
log "github.com/sirupsen/logrus"
)
@ -43,12 +40,10 @@ func registerNarinfoPut(s *Server) {
"output_path": narInfo.StorePath,
})
var pathInfo *storev1pb.PathInfo
// look up the narHash in our temporary map
s.narHashToPathInfoMu.Lock()
pathInfo, found := s.narHashToPathInfo[narInfo.NarHash.SRIString()]
s.narHashToPathInfoMu.Unlock()
s.narDbMu.Lock()
narData, found := s.narDb[narInfo.NarHash.SRIString()]
s.narDbMu.Unlock()
if !found {
log.Error("unable to find referred NAR")
w.WriteHeader(http.StatusBadRequest)
@ -60,10 +55,12 @@ func registerNarinfoPut(s *Server) {
return
}
rootNode := narData.rootNode
// compare fields with what we computed while receiving the NAR file
// NarSize needs to match
if pathInfo.Narinfo.NarSize != narInfo.NarSize {
if narData.narSize != narInfo.NarSize {
log.Error("narsize mismatch")
w.WriteHeader(http.StatusBadRequest)
_, err := w.Write([]byte("unable to parse narinfo"))
@ -73,90 +70,23 @@ func registerNarinfoPut(s *Server) {
return
}
// We know the narhash in the .narinfo matches one of the two narhashes in the partial pathInfo,
// because that's how we found it.
// FUTUREWORK: We can't compare References yet, but it'd be a good idea to
// do reference checking on .nar files server-side during upload.
// We however still need to be parse them, because we store
// the bytes in pathInfo.References, and the full strings in pathInfo.Narinfo.ReferenceNames.
referencesBytes := make([][]byte, 0)
for _, reference := range narInfo.References {
storePath, err := storepath.FromString(reference)
pathInfo, err := importer.GenPathInfo(rootNode, narInfo)
if err != nil {
log.WithError(err).Error("unable to generate PathInfo")
w.WriteHeader(http.StatusInternalServerError)
_, err := w.Write([]byte("unable to generate PathInfo"))
if err != nil {
log.WithField("reference", reference).WithError(err).Error("unable to parse reference")
w.WriteHeader(http.StatusBadRequest)
_, err := w.Write([]byte("unable to parse reference"))
if err != nil {
log.WithError(err).Errorf("unable to write error message to client")
}
return
log.WithError(err).Errorf("unable to write error message to client")
}
referencesBytes = append(referencesBytes, storePath.Digest)
return
}
// assemble the []*storev1pb.NARInfo_Signature{} from narinfo.Signatures.
pbNarinfoSignatures := make([]*storev1pb.NARInfo_Signature, 0)
for _, narinfoSig := range narInfo.Signatures {
log.WithField("pathInfo", pathInfo).Debug("inserted new pathInfo")
pbNarinfoSignatures = append(pbNarinfoSignatures, &storev1pb.NARInfo_Signature{
Name: narinfoSig.Name,
Data: narinfoSig.Data,
})
}
// If everything matches, We will add References, NAR signatures and the
// output path name, and then upload to the pathinfo service.
// We want a copy here, because we don't want to mutate the contents in the lookup table
// until we get things back from the remote store.
pathInfoToUpload := &storev1pb.PathInfo{
Node: nil, // set below
References: referencesBytes,
Narinfo: &storev1pb.NARInfo{
NarSize: pathInfo.Narinfo.NarSize,
NarSha256: pathInfo.Narinfo.NarSha256,
Signatures: pbNarinfoSignatures,
ReferenceNames: narInfo.References,
},
}
// We need to add the basename of the storepath from the .narinfo
// to the pathInfo to be sent.
switch v := (pathInfo.GetNode().GetNode()).(type) {
case *castorev1pb.Node_File:
pathInfoToUpload.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_File{
File: &castorev1pb.FileNode{
Name: []byte(path.Base(narInfo.StorePath)),
Digest: v.File.Digest,
Size: v.File.Size,
Executable: v.File.Executable,
},
},
}
case *castorev1pb.Node_Symlink:
pathInfoToUpload.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_Symlink{
Symlink: &castorev1pb.SymlinkNode{
Name: []byte(path.Base(narInfo.StorePath)),
Target: v.Symlink.Target,
},
},
}
case *castorev1pb.Node_Directory:
pathInfoToUpload.Node = &castorev1pb.Node{
Node: &castorev1pb.Node_Directory{
Directory: &castorev1pb.DirectoryNode{
Name: []byte(path.Base(narInfo.StorePath)),
Digest: v.Directory.Digest,
Size: v.Directory.Size,
},
},
}
}
receivedPathInfo, err := s.pathInfoServiceClient.Put(ctx, pathInfoToUpload)
receivedPathInfo, err := s.pathInfoServiceClient.Put(ctx, pathInfo)
if err != nil {
log.WithError(err).Error("unable to upload pathinfo to service")
w.WriteHeader(http.StatusInternalServerError)
@ -168,8 +98,6 @@ func registerNarinfoPut(s *Server) {
return
}
log.Debugf("received new pathInfo: %v+", receivedPathInfo)
// TODO: update the local temporary pathinfo with this?
log.WithField("pathInfo", receivedPathInfo).Debug("got back PathInfo")
})
}

14
tvix/nar-bridge/pkg/http/server.go
View file

@ -25,11 +25,17 @@ type Server struct {
// When uploading NAR files to a HTTP binary cache, the .nar
// files are uploaded before the .narinfo files.
// We need *both* to be able to fully construct a PathInfo object.
// Keep a in-memory map of narhash(es) (in SRI) to sparse PathInfo.
// Keep a in-memory map of narhash(es) (in SRI) to (unnamed) root node and nar
// size.
// This is necessary until we can ask a PathInfoService for a node with a given
// narSha256.
narHashToPathInfoMu sync.Mutex
narHashToPathInfo map[string]*storev1pb.PathInfo
narDbMu sync.Mutex
narDb map[string]*narData
}
type narData struct {
rootNode *castorev1pb.Node
narSize uint64
}
func New(
@ -64,7 +70,7 @@ func New(
directoryServiceClient: directoryServiceClient,
blobServiceClient: blobServiceClient,
pathInfoServiceClient: pathInfoServiceClient,
narHashToPathInfo: make(map[string]*storev1pb.PathInfo),
narDb: make(map[string]*narData),
}
registerNarPut(s)