maurice/snix
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

chore: Remove buildGo and supporting infrastructure

Browse source 
It's no longer in use.

Change-Id: Ie84ab1d9c75be79a25ee976575660ee1484e17bb
Reviewed-on: https://cl.snix.dev/c/snix/+/30187
Tested-by: besadii
Reviewed-by: Florian Klink <flokli@flokli.de>
This commit is contained in:
adisbladis 2025-03-20 18:58:47 +01:00 • committed by adis bladis
parent d99819280a
commit 0959faf056
52 changed files with 0 additions and 1520 deletions
Download patch file Download diff file Expand all files Collapse all files

2
nix/buildGo/.skip-subtree
View file

@ -1,2 +0,0 @@
Subdirectories of this folder should not be imported since they are
internal to buildGo.nix and incompatible with readTree.

117
nix/buildGo/README.md
View file

@ -1,117 +0,0 @@
buildGo.nix
===========
This is an alternative [Nix][] build system for [Go][]. It supports building Go
libraries and programs.
*Note:* This will probably end up being folded into [Nixery][].
## Background
Most language-specific Nix tooling outsources the build to existing
language-specific build tooling, which essentially means that Nix ends up being
a wrapper around all sorts of external build systems.
However, systems like [Bazel][] take an alternative approach in which the
compiler is invoked directly and the composition of programs and libraries stays
within a single homogeneous build system.
Users don't need to learn per-language build systems and especially for
companies with large monorepo-setups ([like Google][]) this has huge
productivity impact.
This project is an attempt to prove that Nix can be used in a similar style to
build software directly, rather than shelling out to other build systems.
## Example
Given a program layout like this:
```
.
├── lib <-- some library component
│   ├── bar.go
│   └── foo.go
├── main.go <-- program implementation
└── default.nix <-- build instructions
```
The contents of `default.nix` could look like this:
```nix
{ buildGo }:
let
lib = buildGo.package {
name = "somelib";
srcs = [
./lib/bar.go
./lib/foo.go
];
};
in buildGo.program {
name = "my-program";
deps = [ lib ];
srcs = [
./main.go
];
}
```
(If you don't know how to read Nix, check out [nix-1p][])
## Usage
`buildGo` exposes five different functions:
* `buildGo.program`: Build a Go binary out of the specified source files.
| parameter | type | use | required? |
|-----------|-------------------------|------------------------------------------------|-----------|
| `name` | `string` | Name of the program (and resulting executable) | yes |
| `srcs` | `list<path>` | List of paths to source files | yes |
| `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no |
| `x_defs` | `attrs<string, string>` | Attribute set of linker vars (i.e. `-X`-flags) | no |
* `buildGo.package`: Build a Go library out of the specified source files.
| parameter | type | use | required? |
|-----------|--------------|------------------------------------------------|-----------|
| `name` | `string` | Name of the library | yes |
| `srcs` | `list<path>` | List of paths to source files | yes |
| `deps` | `list<drv>` | List of dependencies (i.e. other Go libraries) | no |
| `path` | `string` | Go import path for the resulting library | no |
* `buildGo.external`: Build an externally defined Go library or program.
This function performs analysis on the supplied source code (which
can use the standard Go tooling layout) and creates a tree of all
the packages contained within.
This exists for compatibility with external libraries that were not
defined using buildGo.
| parameter | type | use | required? |
|-----------|----------------|-----------------------------------------------|-----------|
| `path` | `string` | Go import path for the resulting package | yes |
| `src` | `path` | Path to the source **directory** | yes |
| `deps` | `list<drv>` | List of dependencies (i.e. other Go packages) | no |
## Current status
This project is work-in-progress. Crucially it is lacking the following features:
* feature flag parity with Bazel's Go rules
* documentation building
* test execution
There are still some open questions around how to structure some of those
features in Nix.
[Nix]: https://nixos.org/nix/
[Go]: https://golang.org/
[Nixery]: https://github.com/google/nixery
[Bazel]: https://bazel.build/
[like Google]: https://ai.google/research/pubs/pub45424
[nix-1p]: https://github.com/tazjin/nix-1p

157
nix/buildGo/default.nix
View file

@ -1,157 +0,0 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
#
# buildGo provides Nix functions to build Go packages in the style of Bazel's
# rules_go.
{ pkgs ? import <nixpkgs> { }
, ...
}:
let
inherit (builtins)
attrNames
baseNameOf
dirOf
elemAt
filter
listToAttrs
map
match
readDir
replaceStrings
toString;
inherit (pkgs) lib runCommand fetchFromGitHub protobuf symlinkJoin go;
goStdlib = buildStdlib go;
# Helpers for low-level Go compiler invocations
spaceOut = lib.concatStringsSep " ";
includeDepSrc = dep: "-I ${dep}";
includeSources = deps: spaceOut (map includeDepSrc deps);
includeDepLib = dep: "-L ${dep}";
includeLibs = deps: spaceOut (map includeDepLib deps);
srcBasename = src: elemAt (match "([a-z0-9]{32}\-)?(.*\.go)" (baseNameOf src)) 1;
srcCopy = path: src: "cp ${src} $out/${path}/${srcBasename src}";
srcList = path: srcs: lib.concatStringsSep "\n" (map (srcCopy path) srcs);
allDeps = deps: lib.unique (lib.flatten (deps ++ (map (d: d.goDeps) deps)));
xFlags = x_defs: spaceOut (map (k: "-X ${k}=${x_defs."${k}"}") (attrNames x_defs));
# Add an `overrideGo` attribute to a function result that works
# similar to `overrideAttrs`, but is used specifically for the
# arguments passed to Go builders.
makeOverridable = f: orig: (f orig) // {
overrideGo = new: makeOverridable f (orig // (new orig));
};
buildStdlib = go: runCommand "go-stdlib-${go.version}"
{
nativeBuildInputs = [ go ];
} ''
HOME=$NIX_BUILD_TOP/home
mkdir $HOME
goroot="$(go env GOROOT)"
cp -R "$goroot/src" "$goroot/pkg" .
chmod -R +w .
GODEBUG=installgoroot=all GOROOT=$NIX_BUILD_TOP go install -v --trimpath std
mkdir $out
cp -r pkg/*_*/* $out
find $out -name '*.a' | while read -r ARCHIVE_FULL; do
ARCHIVE="''${ARCHIVE_FULL#"$out/"}"
PACKAGE="''${ARCHIVE%.a}"
echo "packagefile $PACKAGE=$ARCHIVE_FULL"
done > $out/importcfg
'';
importcfgCmd = { name, deps, out ? "importcfg" }: ''
echo "# nix buildGo ${name}" > "${out}"
cat "${goStdlib}/importcfg" >> "${out}"
${lib.concatStringsSep "\n" (map (dep: ''
find "${dep}" -name '*.a' | while read -r pkgp; do
relpath="''${pkgp#"${dep}/"}"
pkgname="''${relpath%.a}"
echo "packagefile $pkgname=$pkgp"
done >> "${out}"
'') deps)}
'';
# High-level build functions
# Build a Go program out of the specified files and dependencies.
program = { name, srcs, deps ? [ ], x_defs ? { } }:
let uniqueDeps = allDeps (map (d: d.gopkg) deps);
in runCommand name { } ''
${importcfgCmd { inherit name; deps = uniqueDeps; }}
${go}/bin/go tool compile -o ${name}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p main ${includeSources uniqueDeps} ${spaceOut srcs}
mkdir -p $out/bin
export GOROOT_FINAL=go
${go}/bin/go tool link -o $out/bin/${name} -importcfg=importcfg -buildid nix ${xFlags x_defs} ${includeLibs uniqueDeps} ${name}.a
'';
# Build a Go library assembled out of the specified files.
#
# This outputs both the sources and compiled binary, as both are
# needed when downstream packages depend on it.
package = { name, srcs, deps ? [ ], path ? name, sfiles ? [ ] }:
let
uniqueDeps = allDeps (map (d: d.gopkg) deps);
# The build steps below need to be executed conditionally for Go
# assembly if the analyser detected any *.s files.
#
# This is required for several popular packages (e.g. x/sys).
ifAsm = do: lib.optionalString (sfiles != [ ]) do;
asmBuild = ifAsm ''
${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -gensymabis -o ./symabis ${spaceOut sfiles}
${go}/bin/go tool asm -p ${path} -trimpath $PWD -I $PWD -I ${go}/share/go/pkg/include -D GOOS_linux -D GOARCH_amd64 -o ./asm.o ${spaceOut sfiles}
'';
asmLink = ifAsm "-symabis ./symabis -asmhdr $out/go_asm.h";
asmPack = ifAsm ''
${go}/bin/go tool pack r $out/${path}.a ./asm.o
'';
gopkg = (runCommand "golib-${name}" { } ''
mkdir -p $out/${path}
${srcList path (map (s: "${s}") srcs)}
${asmBuild}
${importcfgCmd { inherit name; deps = uniqueDeps; }}
${go}/bin/go tool compile -pack ${asmLink} -o $out/${path}.a -importcfg=importcfg -trimpath=$PWD -trimpath=${go} -p ${path} ${includeSources uniqueDeps} ${spaceOut srcs}
${asmPack}
'').overrideAttrs (_: {
passthru = {
inherit gopkg;
goDeps = uniqueDeps;
goImportPath = path;
};
});
in
gopkg;
# Build a tree of Go libraries out of an external Go source
# directory that follows the standard Go layout and was not built
# with buildGo.nix.
#
# The derivation for each actual package will reside in an attribute
# named "gopkg", and an attribute named "gobin" for binaries.
external = import ./external { inherit pkgs program package; };
in
{
# Only the high-level builder functions are exposed, but made
# overrideable.
program = makeOverridable program;
package = makeOverridable package;
external = makeOverridable external;
# re-expose the Go version used
inherit go;
}

40
nix/buildGo/example/default.nix
View file

@ -1,40 +0,0 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
# This file provides examples for how to use the various builder
# functions provided by `buildGo`.
#
# The features used in the example are not exhaustive, but should give
# users a quick introduction to how to use buildGo.
let
buildGo = import ../default.nix { };
# Example use of buildGo.package, which creates an importable Go
# package from the specified source files.
examplePackage = buildGo.package {
name = "example";
srcs = [
./lib.go
];
};
# Example use of buildGo.program, which builds an executable using
# the specified name and dependencies (which in turn must have been
# created via buildGo.package etc.)
in
buildGo.program {
name = "example";
srcs = [
./main.go
];
deps = [
examplePackage
];
x_defs = {
"main.Flag" = "successfully";
};
}

9
nix/buildGo/example/lib.go
View file

@ -1,9 +0,0 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
package example
// UUID returns a totally random, carefully chosen UUID
func UUID() string {
return "3640932f-ad40-4bc9-b45d-f504a0f5910a"
}

25
nix/buildGo/example/main.go
View file

@ -1,25 +0,0 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
//
// Package main provides a tiny example program for the Bazel-style
// Nix build system for Go.
package main
import (
"example"
"exampleproto"
"fmt"
)
var Flag string = "unsuccessfully"
func main() {
thing := exampleproto.Thing{
Id: example.UUID(),
KindOfThing: "test thing",
}
fmt.Printf("The thing is a %s with ID %q\n", thing.Id, thing.KindOfThing)
fmt.Printf("The flag has been %s set\n", Flag)
}

112
nix/buildGo/external/default.nix vendored
View file

@ -1,112 +0,0 @@
# Copyright 2019 Google LLC.
# SPDX-License-Identifier: Apache-2.0
{ pkgs, program, package }:
let
inherit (builtins)
elemAt
foldl'
fromJSON
head
length
listToAttrs
readFile
replaceStrings
tail
unsafeDiscardStringContext
throw;
inherit (pkgs) lib runCommand go jq ripgrep;
pathToName = p: replaceStrings [ "/" ] [ "_" ] (toString p);
# Collect all non-vendored dependencies from the Go standard library
# into a file that can be used to filter them out when processing
# dependencies.
stdlibPackages = runCommand "stdlib-pkgs.json" { } ''
export HOME=$PWD
export GOPATH=/dev/null
${go}/bin/go list std | \
${ripgrep}/bin/rg -v 'vendor' | \
${jq}/bin/jq -R '.' | \
${jq}/bin/jq -c -s 'map({key: ., value: true}) | from_entries' \
> $out
'';
analyser = program {
name = "analyser";
srcs = [
./main.go
];
x_defs = {
"main.stdlibList" = "${stdlibPackages}";
};
};
mkset = path: value:
if path == [ ] then { gopkg = value; }
else { "${head path}" = mkset (tail path) value; };
last = l: elemAt l ((length l) - 1);
toPackage = self: src: path: depMap: entry:
let
localDeps = map
(d: lib.attrByPath (d ++ [ "gopkg" ])
(
throw "missing local dependency '${lib.concatStringsSep "." d}' in '${path}'"
)
self)
entry.localDeps;
foreignDeps = map
(d: lib.attrByPath [ d.path ]
(
throw "missing foreign dependency '${d.path}' in '${path}, imported at ${d.position}'"
)
depMap)
entry.foreignDeps;
args = {
srcs = map (f: src + ("/" + f)) entry.files;
deps = localDeps ++ foreignDeps;
};
libArgs = args // {
name = pathToName entry.name;
path = lib.concatStringsSep "/" ([ path ] ++ entry.locator);
sfiles = map (f: src + ("/" + f)) entry.sfiles;
};
binArgs = args // {
name = (last ((lib.splitString "/" path) ++ entry.locator));
};
in
if entry.isCommand then (program binArgs) else (package libArgs);
in
{ src, path, deps ? [ ] }:
let
# Build a map of dependencies (from their import paths to their
# derivation) so that they can be conditionally imported only in
# sub-packages that require them.
depMap = listToAttrs (map
(d: {
name = d.goImportPath;
value = d;
})
(map (d: d.gopkg) deps));
name = pathToName path;
analysisOutput = runCommand "${name}-structure.json" { } ''
${analyser}/bin/analyser -path ${path} -source ${src} > $out
'';
# readFile adds the references of the read in file to the string context for
# Nix >= 2.6 which would break the attribute set construction in fromJSON
analysis = fromJSON (unsafeDiscardStringContext (readFile analysisOutput));
in
lib.fix (self: foldl' lib.recursiveUpdate { } (
map (entry: mkset entry.locator (toPackage self src path depMap entry)) analysis
))

200
nix/buildGo/external/main.go vendored
View file

@ -1,200 +0,0 @@
// Copyright 2019 Google LLC.
// SPDX-License-Identifier: Apache-2.0
// This tool analyses external (i.e. not built with `buildGo.nix`) Go
// packages to determine a build plan that Nix can import.
package main
import (
"encoding/json"
"flag"
"fmt"
"go/build"
"log"
"os"
"path"
"path/filepath"
"strings"
)
// Path to a JSON file describing all standard library import paths.
// This file is generated and set here by Nix during the build
// process.
var stdlibList string
// pkg describes a single Go package within the specified source
// directory.
//
// Return information includes the local (relative from project root)
// and external (none-stdlib) dependencies of this package.
type pkg struct {
Name string `json:"name"`
Locator []string `json:"locator"`
Files []string `json:"files"`
SFiles []string `json:"sfiles"`
LocalDeps [][]string `json:"localDeps"`
ForeignDeps []foreignDep `json:"foreignDeps"`
IsCommand bool `json:"isCommand"`
}
type foreignDep struct {
Path string `json:"path"`
// filename, column and line number of the import, if known
Position string `json:"position"`
}
// findGoDirs returns a filepath.WalkFunc that identifies all
// directories that contain Go source code in a certain tree.
func findGoDirs(at string) ([]string, error) {
dirSet := make(map[string]bool)
err := filepath.Walk(at, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
name := info.Name()
// Skip folders that are guaranteed to not be relevant
if info.IsDir() && (name == "testdata" || name == ".git") {
return filepath.SkipDir
}
// If the current file is a Go file, then the directory is popped
// (i.e. marked as a Go directory).
if !info.IsDir() && strings.HasSuffix(name, ".go") && !strings.HasSuffix(name, "_test.go") {
dirSet[filepath.Dir(path)] = true
}
return nil
})
if err != nil {
return nil, err
}
goDirs := []string{}
for goDir := range dirSet {
goDirs = append(goDirs, goDir)
}
return goDirs, nil
}
// analysePackage loads and analyses the imports of a single Go
// package, returning the data that is required by the Nix code to
// generate a derivation for this package.
func analysePackage(root, source, importpath string, stdlib map[string]bool) (pkg, error) {
ctx := build.Default
ctx.CgoEnabled = false
p, err := ctx.ImportDir(source, build.IgnoreVendor)
if err != nil {
return pkg{}, err
}
local := [][]string{}
foreign := []foreignDep{}
for _, i := range p.Imports {
if stdlib[i] {
continue
}
if i == importpath {
local = append(local, []string{})
} else if strings.HasPrefix(i, importpath+"/") {
local = append(local, strings.Split(strings.TrimPrefix(i, importpath+"/"), "/"))
} else {
// The import positions is a map keyed on the import name.
// The value is a list, presumably because an import can appear
// multiple times in a package. Let’s just take the first one,
// should be enough for a good error message.
firstPos := p.ImportPos[i][0].String()
foreign = append(foreign, foreignDep{Path: i, Position: firstPos})
}
}
prefix := strings.TrimPrefix(source, root+"/")
locator := []string{}
if len(prefix) != len(source) {
locator = strings.Split(prefix, "/")
} else {
// Otherwise, the locator is empty since its the root package and
// no prefix should be added to files.
prefix = ""
}
files := []string{}
for _, f := range p.GoFiles {
files = append(files, path.Join(prefix, f))
}
sfiles := []string{}
for _, f := range p.SFiles {
sfiles = append(sfiles, path.Join(prefix, f))
}
return pkg{
Name: path.Join(importpath, prefix),
Locator: locator,
Files: files,
SFiles: sfiles,
LocalDeps: local,
ForeignDeps: foreign,
IsCommand: p.IsCommand(),
}, nil
}
func loadStdlibPkgs(from string) (pkgs map[string]bool, err error) {
f, err := os.ReadFile(from)
if err != nil {
return
}
err = json.Unmarshal(f, &pkgs)
return
}
func main() {
source := flag.String("source", "", "path to directory with sources to process")
path := flag.String("path", "", "import path for the package")
flag.Parse()
if *source == "" {
log.Fatalf("-source flag must be specified")
}
stdlibPkgs, err := loadStdlibPkgs(stdlibList)
if err != nil {
log.Fatalf("failed to load standard library index from %q: %s\n", stdlibList, err)
}
goDirs, err := findGoDirs(*source)
if err != nil {
log.Fatalf("failed to walk source directory '%s': %s", *source, err)
}
all := []pkg{}
for _, d := range goDirs {
analysed, err := analysePackage(*source, d, *path, stdlibPkgs)
// If the Go source analysis returned "no buildable Go files",
// that directory should be skipped.
//
// This might be due to `+build` flags on the platform and other
// reasons (such as test files).
if _, ok := err.(*build.NoGoError); ok {
continue
}
if err != nil {
log.Fatalf("failed to analyse package at %q: %s", d, err)
}
all = append(all, analysed)
}
j, _ := json.Marshal(all)
fmt.Println(string(j))
}