From ae4d967288dd0cab6f642a5c182665bf6a224eb5 Mon Sep 17 00:00:00 2001 From: Florian Klink Date: Wed, 19 Mar 2025 23:42:36 +0000 Subject: [PATCH] chore(ops): move archivist machine to ops and contrib contrib/ gets the clickhouse patching, the bucket log parsing code and the awscli setup and shell. ops/ gets the machine config itself. Change-Id: If8b8f8cce5ca9c2b4d19e17be9a8b895ac35e84a Reviewed-on: https://cl.snix.dev/c/snix/+/30163 Autosubmit: Florian Klink Tested-by: besadii Reviewed-by: Ryan Lahfa --- {users/flokli => contrib}/archivist/OWNERS | 0 {users/flokli => contrib}/archivist/README.md | 4 +-- .../flokli => contrib}/archivist/default.nix | 1 - .../archivist/parse_bucket_logs.rs | 0 ops/machines/all-systems.nix | 2 ++ .../machines}/archivist-ec2/OWNERS | 0 .../machines/archivist-ec2/default.nix | 12 ++++--- .../archivist-ec2/hardware-configuration.nix | 0 .../parse-bucket-logs-continuously.py | 0 .../profiles => ops/modules}/archivist.nix | 0 ops/nixos.nix | 10 +++++- users/flokli/OWNERS | 1 - users/flokli/nixos/.skip-subtree | 0 users/flokli/nixos/default.nix | 31 ------------------- 14 files changed, 21 insertions(+), 40 deletions(-) rename {users/flokli => contrib}/archivist/OWNERS (100%) rename {users/flokli => contrib}/archivist/README.md (87%) rename {users/flokli => contrib}/archivist/default.nix (99%) rename {users/flokli => contrib}/archivist/parse_bucket_logs.rs (100%) rename {users/flokli/nixos => ops/machines}/archivist-ec2/OWNERS (100%) rename users/flokli/nixos/archivist-ec2/configuration.nix => ops/machines/archivist-ec2/default.nix (72%) rename {users/flokli/nixos => ops/machines}/archivist-ec2/hardware-configuration.nix (100%) rename {users/flokli/nixos => ops/machines}/archivist-ec2/parse-bucket-logs-continuously.py (100%) rename {users/flokli/nixos/profiles => ops/modules}/archivist.nix (100%) delete mode 100644 users/flokli/OWNERS delete mode 100644 users/flokli/nixos/.skip-subtree delete mode 100644 users/flokli/nixos/default.nix diff --git a/users/flokli/archivist/OWNERS b/contrib/archivist/OWNERS similarity index 100% rename from users/flokli/archivist/OWNERS rename to contrib/archivist/OWNERS diff --git a/users/flokli/archivist/README.md b/contrib/archivist/README.md similarity index 87% rename from users/flokli/archivist/README.md rename to contrib/archivist/README.md index 904c87a4e..4176daf6e 100644 --- a/users/flokli/archivist/README.md +++ b/contrib/archivist/README.md @@ -2,7 +2,7 @@ This directory contains various scripts and helpers used for nix-archivist tasks. -It's used from some of the archivist EC2 instance, as well as standalone. +It's used from the archivist EC2 instance, as well as standalone. ## AWS Profile setup There's 2 AWS Accounts, reachable via the nixos.awsapps.com SSO portal. @@ -14,7 +14,7 @@ This is assuming the `archeologist` AWS role in the main NixOS account. This is a separate AWS Account, only for the archivist project. We can assume `AWSAdministratorAccess` in there. -## Machine +## archivist-ec2 Machine The `archivist-ec2` machine currently is deployed in the main NixOS account. It regularly processes S3 bucket logs and dumps them in parquet format into diff --git a/users/flokli/archivist/default.nix b/contrib/archivist/default.nix similarity index 99% rename from users/flokli/archivist/default.nix rename to contrib/archivist/default.nix index 43fc3ec1f..ae28ae1dd 100644 --- a/users/flokli/archivist/default.nix +++ b/contrib/archivist/default.nix @@ -35,7 +35,6 @@ depot.nix.readTree.drvTargets { --prefix PATH : ${pkgs.lib.makeBinPath [ clickhouseLocalFixedAWS ]} ''; - # A shell, by default pointing us to the archivist SSO profile / account by default. shell = pkgs.mkShell { name = "archivist-shell"; diff --git a/users/flokli/archivist/parse_bucket_logs.rs b/contrib/archivist/parse_bucket_logs.rs similarity index 100% rename from users/flokli/archivist/parse_bucket_logs.rs rename to contrib/archivist/parse_bucket_logs.rs diff --git a/ops/machines/all-systems.nix b/ops/machines/all-systems.nix index 275ec6f02..3a9e47b3e 100644 --- a/ops/machines/all-systems.nix +++ b/ops/machines/all-systems.nix @@ -1,6 +1,8 @@ { depot, ... }: (with depot.ops.machines; [ + # Archivist EC2 machine + archivist-ec2 # Gerrit instance gerrit01 # Public-facing services diff --git a/users/flokli/nixos/archivist-ec2/OWNERS b/ops/machines/archivist-ec2/OWNERS similarity index 100% rename from users/flokli/nixos/archivist-ec2/OWNERS rename to ops/machines/archivist-ec2/OWNERS diff --git a/users/flokli/nixos/archivist-ec2/configuration.nix b/ops/machines/archivist-ec2/default.nix similarity index 72% rename from users/flokli/nixos/archivist-ec2/configuration.nix rename to ops/machines/archivist-ec2/default.nix index b4d538f4a..549010758 100644 --- a/users/flokli/nixos/archivist-ec2/configuration.nix +++ b/ops/machines/archivist-ec2/default.nix @@ -1,9 +1,13 @@ -{ depot, pkgs, modulesPath, ... }: +{ depot, pkgs, ... }: # readTree options +{ modulesPath, ... }: # passed by module system +let + mod = name: depot.path.origSrc + ("/ops/modules/" + name); +in { imports = [ "${modulesPath}/virtualisation/amazon-image.nix" - ../profiles/archivist.nix + (mod "archivist.nix") ]; nixpkgs.hostPlatform = "x86_64-linux"; @@ -14,7 +18,7 @@ }; systemd.services.parse-bucket-logs = { - path = [ depot.users.flokli.archivist.parse-bucket-logs ]; + path = [ depot.contrib.archivist.parse-bucket-logs ]; serviceConfig = { Type = "oneshot"; ExecStart = (pkgs.writers.writePython3 "parse-bucket-logs-continuously" @@ -27,7 +31,7 @@ }; environment.systemPackages = [ - depot.users.flokli.archivist.parse-bucket-logs + depot.contrib.archivist.parse-bucket-logs ]; networking.hostName = "archivist-ec2"; diff --git a/users/flokli/nixos/archivist-ec2/hardware-configuration.nix b/ops/machines/archivist-ec2/hardware-configuration.nix similarity index 100% rename from users/flokli/nixos/archivist-ec2/hardware-configuration.nix rename to ops/machines/archivist-ec2/hardware-configuration.nix diff --git a/users/flokli/nixos/archivist-ec2/parse-bucket-logs-continuously.py b/ops/machines/archivist-ec2/parse-bucket-logs-continuously.py similarity index 100% rename from users/flokli/nixos/archivist-ec2/parse-bucket-logs-continuously.py rename to ops/machines/archivist-ec2/parse-bucket-logs-continuously.py diff --git a/users/flokli/nixos/profiles/archivist.nix b/ops/modules/archivist.nix similarity index 100% rename from users/flokli/nixos/profiles/archivist.nix rename to ops/modules/archivist.nix diff --git a/ops/nixos.nix b/ops/nixos.nix index d4528beb8..43bf8d9c3 100644 --- a/ops/nixos.nix +++ b/ops/nixos.nix @@ -58,10 +58,18 @@ in rec { ''; # Systems that should be built in CI + archivistEC2System = nixosFor depot.ops.machines.archivist-ec2; gerrit01System = nixosFor depot.ops.machines.gerrit01; public01System = nixosFor depot.ops.machines.public01; build01System = nixosFor depot.ops.machines.build01; meta01System = nixosFor depot.ops.machines.meta01; nixosSnixCache = nixosFor depot.ops.machines.snix-cache; - meta.ci.targets = [ "gerrit01System" "public01System" "build01System" "meta01System" ]; + + meta.ci.targets = [ + "archivistEC2System" + "gerrit01System" + "public01System" + "build01System" + "meta01System" + ]; } diff --git a/users/flokli/OWNERS b/users/flokli/OWNERS deleted file mode 100644 index 36d9cb265..000000000 --- a/users/flokli/OWNERS +++ /dev/null @@ -1 +0,0 @@ -set noparent diff --git a/users/flokli/nixos/.skip-subtree b/users/flokli/nixos/.skip-subtree deleted file mode 100644 index e69de29bb..000000000 diff --git a/users/flokli/nixos/default.nix b/users/flokli/nixos/default.nix deleted file mode 100644 index dc69d54de..000000000 --- a/users/flokli/nixos/default.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ depot, pkgs, ... }: - -let - # assumes `name` is configured appropriately in your .ssh/config - deployScript = name: sys: pkgs.writeShellScriptBin "deploy-${name}" '' - set -eo pipefail - nix-copy-closure --to ${name} --gzip --use-substitutes ${sys} - ssh ${name} nix-env --profile /nix/var/nix/profiles/system --set ${sys} - ssh ${name} ${sys}/bin/switch-to-configuration switch - ''; - -in -depot.nix.readTree.drvTargets rec { - archivistEc2System = (depot.ops.nixos.nixosFor ({ ... }: { - imports = [ - ./archivist-ec2/configuration.nix - ]; - })).config.system.build.toplevel; - - deploy-archivist-ec2 = (deployScript "archivist-ec2" archivistEc2System); - - deps = (depot.nix.lazy-deps { - deploy-archivist-ec2.attr = "users.flokli.nixos.deploy-archivist-ec2"; - aws.attr = "third_party.nixpkgs.awscli"; - }); - - shell = pkgs.mkShell { - name = "flokli-nixos-shell"; - packages = [ deps ]; - }; -}