feat(*): initialize new Snix infrastructure

Co-Authored-By: edef <edef@edef.eu>
Co-Authored-by: Ryan Lahfa <raito@lix.systems>
Change-Id: Ica1cda177a236814de900f50a8a61d288f58f519
This commit is contained in:
Florian Klink 2025-01-06 01:06:47 +01:00
parent 067eff3427
commit a52ea3675c
124 changed files with 27723 additions and 1631 deletions

View file

@ -1,6 +1,12 @@
{ depot, ... }:
(with depot.ops.machines; [
meta01
# Gerrit instance
gerrit01
# Public-facing services
public01
# Build machine
build01
# Observability stack and internal software
meta01
])

View file

@ -0,0 +1,94 @@
{ depot, lib, pkgs, ... }: # readTree options
{ config, ... }: # passed by module system
let
mod = name: depot.path.origSrc + ("/ops/modules/" + name);
in
{
imports = [
(mod "o11y/agent.nix")
(mod "snix-buildkite.nix")
(mod "known-hosts.nix")
(depot.third_party.agenix.src + "/modules/age.nix")
];
# Machine model taken from project Floral.
boot.isContainer = true;
# XXX: There's currently no way to remove the "problematic" entries (trying
# to override the /proc, /sys, /dev, ... mounts from systemd-nspawn) while
# also keeping the entry for the wrappers dir.
boot.specialFileSystems = lib.mkForce {
"/run/wrappers" = {
fsType = "tmpfs";
options = [ "nodev" "mode=755" "size=${config.security.wrapperDirSize}" ];
};
};
services.depot.buildkite = {
enable = true;
agentCount = 32;
};
boot.loader.initScript.enable = true;
system.switch.enableNg = false;
nix.package = pkgs.nix_2_3;
networking = {
useNetworkd = true;
useHostResolvConf = false;
hostName = "build01";
domain = "infra.snix.dev";
nameservers = [
"2001:4860:4860::6464"
"2001:4860:4860::64"
];
interfaces.host0.ipv6 = {
addresses = [
{ address = "2001:bc8:38ee:100:7000::20"; prefixLength = 64; }
];
routes = [
{ address = "64:ff9b::"; via = "2001:bc8:38ee:100::100"; prefixLength = 96; }
];
};
firewall.allowPing = true;
};
age.secrets =
let
secretFile = name: depot.ops.secrets."${name}.age";
in
{
buildkite-agent-token = {
file = secretFile "buildkite-agent-token";
mode = "0440";
group = "buildkite-agents";
};
buildkite-private-key = {
file = secretFile "buildkite-ssh-private-key";
mode = "0440";
group = "buildkite-agents";
};
buildkite-besadii-config = {
file = secretFile "buildkite-besadii-config";
mode = "0440";
group = "buildkite-agents";
};
buildkite-graphql-token = {
file = secretFile "buildkite-graphql-token";
mode = "0440";
group = "buildkite-agents";
};
};
services.openssh.enable = true;
time.timeZone = "UTC";
users.users.root.openssh.authorizedKeys.keys = with depot.users; flokli.keys.all ++ edef.keys.all ++ raito.keys.all;
users.groups.kvm = { };
users.users.root.extraGroups = [ "kvm" ];
system.stateVersion = "25.05";
}

View file

@ -0,0 +1,122 @@
{ depot, lib, pkgs, ... }: # readTree options
{ config, ... }: # passed by module system
let
mod = name: depot.path.origSrc + ("/ops/modules/" + name);
in
{
imports = [
./disko.nix
(mod "hetzner-cloud.nix")
(mod "restic.nix")
(mod "o11y/agent.nix")
(mod "gerrit-autosubmit.nix")
(mod "monorepo-gerrit.nix")
(mod "www/cl.snix.dev.nix")
(mod "known-hosts.nix")
(depot.third_party.agenix.src + "/modules/age.nix")
(depot.third_party.disko.src + "/module.nix")
];
infra.hardware.hetzner-cloud = {
enable = true;
ipv6 = "2a01:4f8:c17:6188::1/64";
};
networking = {
hostName = "gerrit01";
domain = "infra.snix.dev";
};
# Disable background git gc system-wide, as it has a tendency to break CI.
environment.etc."gitconfig".source = pkgs.writeText "gitconfig" ''
[gc]
autoDetach = false
'';
time.timeZone = "UTC";
programs.mtr.enable = true;
programs.mosh.enable = true;
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false;
KbdInteractiveAuthentication = false;
};
};
# Automatically collect garbage from the Nix store.
services.depot.automatic-gc = {
enable = true;
interval = "daily";
diskThreshold = 5; # GiB
maxFreed = 3; # GiB
preserveGenerations = "30d";
};
age.secrets =
let
secretFile = name: depot.ops.secrets."${name}.age";
in
{
gerrit-oauth-secret.file = secretFile "gerrit-oauth-secret";
gerrit-replication-key.file = secretFile "gerrit-replication-key";
gerrit-autosubmit.file = secretFile "gerrit-autosubmit";
gerrit-besadii-config = {
file = secretFile "buildkite-besadii-config";
owner = "git";
};
restic-repository-password.file = secretFile "restic-repository-password";
restic-bucket-credentials.file = secretFile "restic-bucket-credentials";
};
services.depot = {
gerrit-autosubmit.enable = true;
restic.enable = true;
};
services.fail2ban.enable = true;
environment.systemPackages = (with pkgs; [
bat
bb
curl
direnv
fd
git
htop
hyperfine
jq
nano
nvd
ripgrep
tree
unzip
vim
]) ++ (with depot; [
ops.deploy-machine
]);
# Required for prometheus to be able to scrape stats
services.nginx.statusPage = true;
users = {
# Set up a user & group for git shenanigans
groups.git = { };
users.git = {
group = "git";
isSystemUser = true;
createHome = true;
home = "/var/lib/git";
};
users.root.openssh.authorizedKeys.keys = with depot.users; flokli.keys.all ++ edef.keys.all ++ raito.keys.all;
};
boot.initrd.systemd.enable = true;
zramSwap.enable = true;
system.stateVersion = "25.05";
}

View file

@ -0,0 +1,84 @@
let
disk = "/dev/sda";
targetFsType = "xfs";
swapSizeInGb = 16;
in
{
disko.devices = {
disk = {
${disk} = {
device = "${disk}";
type = "disk";
content = {
type = "gpt";
partitions = {
ESP = {
size = "1G";
type = "EF00";
priority = 100;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [ "umask=0077" ];
};
};
PRIMARY = {
# Take the next available range.
start = "0";
end = "-${toString swapSizeInGb}G";
content = {
type = "lvm_pv";
vg = "vg_${targetFsType}";
};
};
SWAP = {
# Start from the SWAP area.
start = "-${toString swapSizeInGb}G";
size = "100%";
content = {
type = "swap";
};
};
};
};
};
};
lvm_vg = {
"vg_${targetFsType}" = {
type = "lvm_vg";
lvs = {
ROOT = {
name = "ROOT";
size = "2G";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/";
};
};
NIX = {
name = "NIX";
size = "40%FREE";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/nix";
mountOptions = [ "noatime" ];
};
};
VAR = {
name = "VAR";
size = "100%FREE";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/var";
mountOptions = [ "noatime" ];
};
};
};
};
};
};
}

View file

@ -0,0 +1,145 @@
{ depot, lib, pkgs, ... }: # readTree options
{ config, ... }: # passed by module system
let
mod = name: depot.path.origSrc + ("/ops/modules/" + name);
in
{
imports = [
./disko.nix
(mod "hetzner-cloud.nix")
(mod "o11y/agent.nix")
(mod "o11y/mimir.nix")
(mod "o11y/loki.nix")
(mod "o11y/tempo.nix")
(mod "o11y/alertmanager-irc-relay.nix")
(mod "known-hosts.nix")
(mod "clbot.nix")
(mod "www/mimir.snix.dev.nix")
(mod "www/loki.snix.dev.nix")
(mod "www/tempo.snix.dev.nix")
(depot.third_party.agenix.src + "/modules/age.nix")
(depot.third_party.disko.src + "/module.nix")
];
infra.hardware.hetzner-cloud = {
enable = true;
ipv6 = "2a01:4f8:c013:4a58::1/64";
};
networking = {
hostName = "meta01";
domain = "infra.snix.dev";
};
time.timeZone = "UTC";
programs.mtr.enable = true;
programs.mosh.enable = true;
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false;
KbdInteractiveAuthentication = false;
};
};
# Automatically collect garbage from the Nix store.
services.depot.automatic-gc = {
enable = true;
interval = "daily";
diskThreshold = 5; # GiB
maxFreed = 3; # GiB
preserveGenerations = "30d";
};
services.depot = {
# TODO: make it possible to do `alertmanager.enable = true;`
prometheus.enable = true;
loki.enable = true;
tempo.enable = true;
clbot = {
enable = false;
channels = {
"#snix" = { };
flags = {
gerrit_host = "cl.snix.dev:29418";
gerrit_ssh_auth_username = "clbot";
# gerrit_ssh_auth_key = config.age.secrets.clbot-ssh-private-key.path;
irc_server = "irc.hackint.org:6697";
irc_tls = true;
irc_user = "snixbot";
irc_nick = "snixbot";
notify_branches = "canon,refs/meta/config";
notify_repo = "snix";
irc_pass = "$CLBOT_PASS";
};
};
};
};
networking.nftables.enable = true;
networking.firewall.extraInputRules = ''
# Prometheus, Loki, Tempo
ip6 saddr { 2a01:4f8:c013:3e62::1 } tcp dport { 9009, 9090, 9190 } accept
ip saddr { 49.13.70.233 } tcp dport { 9009, 9090, 9190 } accept
'';
age.secrets =
let
secretFile = name: depot.ops.secrets."${name}.age";
in
{
mimir-environment.file = secretFile "mimir-environment";
# Yes, they are literally the same: Hetzner Cloud has no support for per-bucket keys.
loki-environment.file = secretFile "mimir-environment";
tempo-environment.file = secretFile "mimir-environment";
metrics-push-htpasswd.file = secretFile "metrics-push-htpasswd";
metrics-push-htpasswd.owner = "nginx";
mimir-webhook-url.file = secretFile "mimir-webhook-url";
alertmanager-irc-relay-environment.file = secretFile "alertmanager-irc-relay-environment";
restic-repository-password.file = secretFile "restic-repository-password";
restic-bucket-credentials.file = secretFile "restic-bucket-credentials";
};
services.fail2ban.enable = true;
environment.systemPackages = (with pkgs; [
bat
bb
curl
direnv
fd
git
htop
hyperfine
jq
nano
nvd
ripgrep
tree
unzip
vim
]) ++ (with depot; [
ops.deploy-machine
]);
# Required for prometheus to be able to scrape stats
services.nginx.statusPage = true;
users = {
users.root.openssh.authorizedKeys.keys = with depot.users; flokli.keys.all ++ edef.keys.all ++ raito.keys.all;
};
boot.initrd.systemd.enable = true;
zramSwap.enable = true;
system.stateVersion = "25.05";
}

View file

@ -0,0 +1,84 @@
let
disk = "/dev/sda";
targetFsType = "xfs";
swapSizeInGb = 8;
in
{
disko.devices = {
disk = {
${disk} = {
device = "${disk}";
type = "disk";
content = {
type = "gpt";
partitions = {
ESP = {
size = "1G";
type = "EF00";
priority = 100;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [ "umask=0077" ];
};
};
PRIMARY = {
# Take the next available range.
start = "0";
end = "-${toString swapSizeInGb}G";
content = {
type = "lvm_pv";
vg = "vg_${targetFsType}";
};
};
SWAP = {
# Start from the SWAP area.
start = "-${toString swapSizeInGb}G";
size = "100%";
content = {
type = "swap";
};
};
};
};
};
};
lvm_vg = {
"vg_${targetFsType}" = {
type = "lvm_vg";
lvs = {
ROOT = {
name = "ROOT";
size = "2G";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/";
};
};
NIX = {
name = "NIX";
size = "40%FREE";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/nix";
mountOptions = [ "noatime" ];
};
};
VAR = {
name = "VAR";
size = "100%FREE";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/var";
mountOptions = [ "noatime" ];
};
};
};
};
};
};
}

View file

@ -0,0 +1,206 @@
{ depot, lib, pkgs, ... }: # readTree options
{ config, ... }: # passed by module system
let
mod = name: depot.path.origSrc + ("/ops/modules/" + name);
in
{
imports = [
./disko.nix
(mod "hetzner-cloud.nix")
(mod "forgejo.nix")
(mod "restic.nix")
# (mod "stalwart.nix")
# Automatically enable metric and log collection.
(mod "o11y/agent.nix")
(mod "o11y/grafana.nix")
(mod "www/status.snix.dev.nix")
(mod "www/auth.snix.dev.nix")
(mod "www/git.snix.dev.nix")
# (mod "www/mail.snix.dev.nix")
(mod "known-hosts.nix")
(depot.third_party.agenix.src + "/modules/age.nix")
(depot.third_party.disko.src + "/module.nix")
];
infra.hardware.hetzner-cloud = {
enable = true;
ipv6 = "2a01:4f8:c013:3e62::1/64";
# Additional IPs.
floatingIPs = [
"49.12.112.149/32"
];
};
networking = {
hostName = "public01";
domain = "infra.snix.dev";
};
time.timeZone = "UTC";
programs.mtr.enable = true;
programs.mosh.enable = true;
services.openssh = {
enable = true;
settings = {
PasswordAuthentication = false;
KbdInteractiveAuthentication = false;
};
};
services.depot = {
# Automatically collect garbage from the Nix store.
automatic-gc = {
enable = true;
interval = "daily";
diskThreshold = 5; # GiB
maxFreed = 3; # GiB
preserveGenerations = "30d";
};
forgejo = {
enable = true;
domain = "git.snix.dev";
};
grafana.enable = true;
# stalwart = {
# enable = true;
# mailDomain = "mail.snix.dev";
# };
# Configure backups to Hetzner Cloud
restic = {
enable = true;
paths = [
"/var/backup/postgresql"
"/var/backup/mysql"
"/var/lib/grafana"
"/var/lib/forgejo"
];
};
};
services.postgresqlBackup = {
enable = true;
databases = [
"keycloak"
];
};
services.mysqlBackup = {
enable = true;
databases = [
"forgejo"
];
};
services.keycloak = {
enable = true;
settings = {
http-port = 9091;
hostname = "auth.snix.dev";
proxy-headers = "xforwarded";
http-enabled = true;
# https://www.keycloak.org/docs/latest/server_admin/#_fine_grain_permissions
features = "admin-fine-grained-authz";
};
# This will be immediately changed, so no harm in having it here.
# It's just a one-time-use random set of characters.
initialAdminPassword = "TUxLWjndUZQGQ0A3ws0LfUs1DYRdAVcK";
database = {
type = "postgresql";
createLocally = true;
passwordFile = config.age.secrets.keycloak-db-password.path;
};
};
systemd.services.keycloak.serviceConfig.Environment = [
# https://bugs.openjdk.org/browse/JDK-8170568 someday… !
"JAVA_OPTS_APPEND=-Djava.net.preferIPv6Addresses=system"
];
age.secrets =
let
secretFile = name: depot.ops.secrets."${name}.age";
in
{
forgejo-oauth-secret = {
file = secretFile "forgejo-oauth-secret";
mode = "0440";
group = "git";
};
grafana-oauth-secret = {
file = secretFile "grafana-oauth-secret";
mode = "0440";
owner = "grafana";
};
keycloak-db-password.file = secretFile "keycloak-db-password";
restic-repository-password.file = secretFile "restic-repository-password";
restic-bucket-credentials.file = secretFile "restic-bucket-credentials";
};
# Start the Gerrit->IRC bot
# services.depot.clbot = {
# enable = true;
# channels = {
# "#snix-dev" = { };
# };
# # See //fun/clbot for details.
# flags = {
# gerrit_host = "cl.tvl.fyi:29418";
# gerrit_ssh_auth_username = "clbot";
# gerrit_ssh_auth_key = config.age.secretsDir + "/clbot-ssh";
# irc_server = "localhost:${toString config.services.znc.config.Listener.l.Port}";
# irc_user = "tvlbot";
# irc_nick = "tvlbot";
# notify_branches = "canon,refs/meta/config";
# notify_repo = "depot";
# # This secret is read from an environment variable, which is
# # populated by a systemd EnvironmentFile.
# irc_pass = "$CLBOT_PASS";
# };
# };
services.fail2ban.enable = true;
environment.systemPackages = (with pkgs; [
bat
bb
curl
direnv
fd
git
htop
hyperfine
jq
nano
nvd
ripgrep
tree
unzip
vim
]) ++ (with depot; [
ops.deploy-machine
]);
# Required for prometheus to be able to scrape stats
services.nginx.statusPage = true;
users = {
users.root.openssh.authorizedKeys.keys = with depot.users; flokli.keys.all ++ edef.keys.all ++ raito.keys.all;
};
boot.initrd.systemd.enable = true;
zramSwap.enable = true;
system.stateVersion = "25.05";
}

View file

@ -0,0 +1,84 @@
let
disk = "/dev/sda";
targetFsType = "xfs";
swapSizeInGb = 16;
in
{
disko.devices = {
disk = {
${disk} = {
device = "${disk}";
type = "disk";
content = {
type = "gpt";
partitions = {
ESP = {
size = "1G";
type = "EF00";
priority = 100;
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
mountOptions = [ "umask=0077" ];
};
};
PRIMARY = {
# Take the next available range.
start = "0";
end = "-${toString swapSizeInGb}G";
content = {
type = "lvm_pv";
vg = "vg_${targetFsType}";
};
};
SWAP = {
# Start from the SWAP area.
start = "-${toString swapSizeInGb}G";
size = "100%";
content = {
type = "swap";
};
};
};
};
};
};
lvm_vg = {
"vg_${targetFsType}" = {
type = "lvm_vg";
lvs = {
ROOT = {
name = "ROOT";
size = "2G";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/";
};
};
NIX = {
name = "NIX";
size = "40%FREE";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/nix";
mountOptions = [ "noatime" ];
};
};
VAR = {
name = "VAR";
size = "100%FREE";
content = {
type = "filesystem";
format = targetFsType;
mountpoint = "/var";
mountOptions = [ "noatime" ];
};
};
};
};
};
};
}