feat(tvix/store/protos): add Export
Export will traverse a given PathInfo structure, and write the contents in NAR format to the passed Writer. It uses directoryLookupFn and blobLookupFn to resolve references. This is being moved over from nar-bridge. We need to keep the code there around until we can bump go.mod to storev1 with this merged, but the tests can already be moved entirely. Change-Id: Ie0de3077b09344cafa00ff1e2ddb8b52e9e631bc Reviewed-on: https://cl.tvl.fyi/c/depot/+/9602 Tested-by: BuildkiteCI Reviewed-by: Brian McGee <brian@bmcgee.ie> Autosubmit: flokli <flokli@flokli.de>
This commit is contained in:
Florian Klink
clbot
parent
6fe34b7ba0
commit
e6ba84ea50
10 changed files with 387 additions and 79 deletions
|
|
@ -1,220 +0,0 @@
|
|||
package exporter_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
castorev1pb "code.tvl.fyi/tvix/castore/protos"
|
||||
"code.tvl.fyi/tvix/nar-bridge/pkg/exporter"
|
||||
"code.tvl.fyi/tvix/nar-bridge/pkg/importer"
|
||||
storev1pb "code.tvl.fyi/tvix/store/protos"
|
||||
"github.com/stretchr/testify/require"
|
||||
"lukechampine.com/blake3"
|
||||
)
|
||||
|
||||
func mustDirectoryDigest(d *castorev1pb.Directory) []byte {
|
||||
dgst, err := d.Digest()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return dgst
|
||||
}
|
||||
|
||||
func mustBlobDigest(r io.Reader) []byte {
|
||||
hasher := blake3.New(32, nil)
|
||||
_, err := io.Copy(hasher, r)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return hasher.Sum([]byte{})
|
||||
}
|
||||
|
||||
func TestSymlink(t *testing.T) {
|
||||
pathInfo := &storev1pb.PathInfo{
|
||||
|
||||
Node: &castorev1pb.Node{
|
||||
Node: &castorev1pb.Node_Symlink{
|
||||
Symlink: &castorev1pb.SymlinkNode{
|
||||
Name: []byte("doesntmatter"),
|
||||
Target: []byte("/nix/store/somewhereelse"),
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := exporter.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) {
|
||||
panic("no directories expected")
|
||||
}, func([]byte) (io.ReadCloser, error) {
|
||||
panic("no files expected")
|
||||
})
|
||||
require.NoError(t, err, "exporter shouldn't fail")
|
||||
|
||||
f, err := os.Open("../../testdata/symlink.nar")
|
||||
require.NoError(t, err)
|
||||
|
||||
bytesExpected, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match")
|
||||
}
|
||||
|
||||
func TestRegular(t *testing.T) {
|
||||
// The blake3 digest of the 0x01 byte.
|
||||
BLAKE3_DIGEST_0X01 := []byte{
|
||||
0x48, 0xfc, 0x72, 0x1f, 0xbb, 0xc1, 0x72, 0xe0, 0x92, 0x5f, 0xa2, 0x7a, 0xf1, 0x67, 0x1d,
|
||||
0xe2, 0x25, 0xba, 0x92, 0x71, 0x34, 0x80, 0x29, 0x98, 0xb1, 0x0a, 0x15, 0x68, 0xa1, 0x88,
|
||||
0x65, 0x2b,
|
||||
}
|
||||
|
||||
pathInfo := &storev1pb.PathInfo{
|
||||
Node: &castorev1pb.Node{
|
||||
Node: &castorev1pb.Node_File{
|
||||
File: &castorev1pb.FileNode{
|
||||
Name: []byte("doesntmatter"),
|
||||
Digest: BLAKE3_DIGEST_0X01,
|
||||
Size: 1,
|
||||
Executable: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := exporter.Export(&buf, pathInfo, func([]byte) (*castorev1pb.Directory, error) {
|
||||
panic("no directories expected")
|
||||
}, func(blobRef []byte) (io.ReadCloser, error) {
|
||||
if !bytes.Equal(blobRef, BLAKE3_DIGEST_0X01) {
|
||||
panic("unexpected blobref")
|
||||
}
|
||||
return io.NopCloser(bytes.NewBuffer([]byte{0x01})), nil
|
||||
})
|
||||
require.NoError(t, err, "exporter shouldn't fail")
|
||||
|
||||
f, err := os.Open("../../testdata/onebyteregular.nar")
|
||||
require.NoError(t, err)
|
||||
|
||||
bytesExpected, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match")
|
||||
}
|
||||
|
||||
func TestEmptyDirectory(t *testing.T) {
|
||||
// construct empty directory node this refers to
|
||||
emptyDirectory := &castorev1pb.Directory{
|
||||
Directories: []*castorev1pb.DirectoryNode{},
|
||||
Files: []*castorev1pb.FileNode{},
|
||||
Symlinks: []*castorev1pb.SymlinkNode{},
|
||||
}
|
||||
emptyDirectoryDigest := mustDirectoryDigest(emptyDirectory)
|
||||
|
||||
pathInfo := &storev1pb.PathInfo{
|
||||
Node: &castorev1pb.Node{
|
||||
Node: &castorev1pb.Node_Directory{
|
||||
Directory: &castorev1pb.DirectoryNode{
|
||||
Name: []byte("doesntmatter"),
|
||||
Digest: emptyDirectoryDigest,
|
||||
Size: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
|
||||
err := exporter.Export(&buf, pathInfo, func(directoryRef []byte) (*castorev1pb.Directory, error) {
|
||||
if !bytes.Equal(directoryRef, emptyDirectoryDigest) {
|
||||
panic("unexpected directoryRef")
|
||||
}
|
||||
return emptyDirectory, nil
|
||||
}, func([]byte) (io.ReadCloser, error) {
|
||||
panic("no files expected")
|
||||
})
|
||||
require.NoError(t, err, "exporter shouldn't fail")
|
||||
|
||||
f, err := os.Open("../../testdata/emptydirectory.nar")
|
||||
require.NoError(t, err)
|
||||
|
||||
bytesExpected, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
require.Equal(t, bytesExpected, buf.Bytes(), "expected nar contents to match")
|
||||
}
|
||||
|
||||
func TestFull(t *testing.T) {
|
||||
// We pipe nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar to the exporter,
|
||||
// and store all the file contents and directory objects received in two hashmaps.
|
||||
// We then feed it to the writer, and test we come up with the same NAR file.
|
||||
|
||||
f, err := os.Open("../../testdata/nar_1094wph9z4nwlgvsd53abfz8i117ykiv5dwnq9nnhz846s7xqd7d.nar")
|
||||
require.NoError(t, err)
|
||||
|
||||
narContents, err := io.ReadAll(f)
|
||||
require.NoError(t, err)
|
||||
|
||||
blobsMap := make(map[string][]byte, 0)
|
||||
directoriesMap := make(map[string]*castorev1pb.Directory)
|
||||
|
||||
pathInfo, err := importer.Import(
|
||||
context.Background(),
|
||||
bytes.NewBuffer(narContents),
|
||||
func(blobReader io.Reader) ([]byte, error) {
|
||||
// read in contents, we need to put it into filesMap later.
|
||||
contents, err := io.ReadAll(blobReader)
|
||||
require.NoError(t, err)
|
||||
|
||||
dgst := mustBlobDigest(bytes.NewReader(contents))
|
||||
|
||||
// put it in filesMap
|
||||
blobsMap[base64.StdEncoding.EncodeToString(dgst)] = contents
|
||||
|
||||
return dgst, nil
|
||||
},
|
||||
func(directory *castorev1pb.Directory) ([]byte, error) {
|
||||
dgst := mustDirectoryDigest(directory)
|
||||
|
||||
directoriesMap[base64.StdEncoding.EncodeToString(dgst)] = directory
|
||||
return dgst, nil
|
||||
},
|
||||
)
|
||||
|
||||
require.NoError(t, err)
|
||||
|
||||
// done populating everything, now actually test the export :-)
|
||||
var buf bytes.Buffer
|
||||
err = exporter.Export(
|
||||
&buf,
|
||||
pathInfo,
|
||||
func(directoryDgst []byte) (*castorev1pb.Directory, error) {
|
||||
d, found := directoriesMap[base64.StdEncoding.EncodeToString(directoryDgst)]
|
||||
if !found {
|
||||
panic(fmt.Sprintf("directory %v not found", base64.StdEncoding.EncodeToString(directoryDgst)))
|
||||
}
|
||||
return d, nil
|
||||
},
|
||||
func(blobDgst []byte) (io.ReadCloser, error) {
|
||||
blobContents, found := blobsMap[base64.StdEncoding.EncodeToString(blobDgst)]
|
||||
if !found {
|
||||
panic(fmt.Sprintf("blob %v not found", base64.StdEncoding.EncodeToString(blobDgst)))
|
||||
}
|
||||
return io.NopCloser(bytes.NewReader(blobContents)), nil
|
||||
},
|
||||
)
|
||||
|
||||
require.NoError(t, err, "exporter shouldn't fail")
|
||||
require.Equal(t, narContents, buf.Bytes())
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue